From 0c59ba4cfd639ea5c1702eca5daf12f83438d6c5 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Thu, 3 Sep 2026 15:30:23 +0100 Subject: [PATCH 1/2] Parallelise and merge ci jobs, make sure rust dependencies are shared everywhere Signed-off-by: Robert Kruszewski --- .github/actions/setup-prebuild/action.yml | 12 + .github/actions/setup-rust/action.yml | 9 + .github/workflows/ci.yml | 464 +++++++++++----------- .github/workflows/compat-validation.yml | 11 +- .github/workflows/cuda.yaml | 83 ++-- .github/workflows/publish-dry-runs.yml | 43 +- .github/workflows/rust-instrumented.yml | 55 +-- 7 files changed, 366 insertions(+), 311 deletions(-) diff --git a/.github/actions/setup-prebuild/action.yml b/.github/actions/setup-prebuild/action.yml index 2b590d5fe8f..0fd897fde41 100644 --- a/.github/actions/setup-prebuild/action.yml +++ b/.github/actions/setup-prebuild/action.yml @@ -156,6 +156,18 @@ runs: shell: bash run: sccache --start-server + # sccache does not cache Cargo's registry, build-script outputs, or linked dependency artifacts. + # Keep one cache key across CI jobs so each fresh VM can reuse those expensive inputs too. + # Only develop publishes caches; PRs restore them without racing to populate the same key. + - name: Restore Cargo dependency cache + if: github.repository == 'vortex-data/vortex' && inputs.enable-sccache == 'true' + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + shared-key: ci-dependencies + add-job-id-key: "false" + cache-bin: "false" + save-if: ${{ github.ref == 'refs/heads/develop' }} + # Fallback path: full setup for forks - name: Full Rust setup if: github.repository != 'vortex-data/vortex' diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml index 0f11c96043b..38c4df38c6e 100644 --- a/.github/actions/setup-rust/action.yml +++ b/.github/actions/setup-rust/action.yml @@ -80,6 +80,15 @@ runs: shell: bash run: sccache --start-server + - name: Restore Cargo dependency cache + if: inputs.enable-sccache == 'true' + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + shared-key: ci-dependencies + add-job-id-key: "false" + cache-bin: "false" + save-if: ${{ github.ref == 'refs/heads/develop' }} + - name: Install Protoc (for lance-encoding build step) if: runner.os != 'Windows' uses: ./.github/actions/setup-protoc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c647a95d42e..8be6fdae7a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,11 +72,12 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 10 steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -104,11 +105,12 @@ jobs: RUST_LOG: "info,maturin=off,uv=debug" MATURIN_PEP517_ARGS: "--profile ci" steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -118,18 +120,19 @@ jobs: uv run --all-packages pytest --benchmark-disable -n auto test/ working-directory: vortex-python/ - # The datasets integration relies on private `datasets` APIs, so exercise the oldest - # supported release in addition to the locked one. - - name: Pytest - Vortex (datasets 4.x floor) - run: | - uv run --all-packages --with 'datasets==4.0.0' --with 'huggingface-hub<1' \ - pytest --benchmark-disable test/test_hf_datasets.py - working-directory: vortex-python/ + - parallel: + # The datasets integration relies on private `datasets` APIs, so exercise the oldest + # supported release in addition to the locked one. + - name: Pytest - Vortex (datasets 4.x floor) + run: | + uv run --all-packages --with 'datasets==4.0.0' --with 'huggingface-hub<1' \ + pytest --benchmark-disable test/test_hf_datasets.py + working-directory: vortex-python/ - - name: Check Python documentation - run: | - uv run --all-packages make check - working-directory: docs/ + - name: Check Python documentation + run: | + uv run --all-packages make check + working-directory: docs/ # Keep this last: setup-benchmark.sh confines everything except the wrapped # benchmark to the housekeeping CPUs, so any step after it runs on few cores. @@ -185,23 +188,25 @@ jobs: RUST_LOG: "info,maturin=off,uv=debug" MATURIN_PEP517_ARGS: "--profile ci" steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - components: cargo - enable-sccache: "true" + - parallel: + - uses: runs-on/action@v2 + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + components: cargo + enable-sccache: "true" + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 + with: + sync: false - name: Pin rustup proxy to repository toolchain run: | TOOLCHAIN="$(grep '^channel' rust-toolchain.toml | cut -d '"' -f 2)" echo "RUSTUP_TOOLCHAIN=$TOOLCHAIN" >> "$GITHUB_ENV" - - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 - with: - sync: false - name: Pytest - PyVortex CUDA bridge run: | @@ -218,11 +223,12 @@ jobs: && format('runs-on={0}/runner=amd64-ci-small/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-docs', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -244,19 +250,12 @@ jobs: ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner={1}/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag={2}', github.run_id, matrix.config.runner, matrix.config.name) || 'ubuntu-latest' }} - env: - # disable lints for build, they will be caught in Rust lint job. - RUSTFLAGS: "-A warnings" strategy: fail-fast: false matrix: config: - - name: "all-features" + - name: "native feature sets" runner: amd64-ci-large - args: "--all-features --all-targets" - - name: "default features" - runner: amd64-ci-large - args: "--all-targets" - name: "with tokio dispatcher" runner: amd64-ci-small # Only build the crates that have the tokio features, not re-building other crates with no-default-features @@ -268,22 +267,44 @@ jobs: rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"unsupported\"'" args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-python-cuda --exclude vortex-sqllogictest --exclude vortex-parquet-variant" steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" targets: ${{ matrix.config.target }} + - parallel: + - name: Build all features + if: matrix.config.name == 'native feature sets' + uses: ./.github/actions/check-rebuild + with: + command: >- + CARGO_TARGET_DIR=${{ runner.temp }}/vortex-build/all-features + cargo hack build --jobs "$(( ($(nproc) + 1) / 2 ))" --profile ci --locked + --all-features --all-targets --ignore-private + - name: Build default features + if: matrix.config.name == 'native feature sets' + uses: ./.github/actions/check-rebuild + with: + command: >- + CARGO_TARGET_DIR=${{ runner.temp }}/vortex-build/default-features + cargo hack build --jobs "$(( ($(nproc) + 1) / 2 ))" --profile ci --locked + --all-targets --ignore-private - uses: ./.github/actions/check-rebuild + if: matrix.config.name != 'native feature sets' with: command: "${{matrix.config.env.rustflags}} cargo hack build --profile ci --locked ${{matrix.config.args}} --ignore-private" - name: "Make sure no files changed after build" run: | git status --porcelain test -z "$(git status --porcelain)" + - name: sccache stats + if: always() && github.repository == 'vortex-data/vortex' + run: sccache --show-stats check-min-deps: name: "Check build with minimal dependencies" @@ -293,11 +314,12 @@ jobs: && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-min-deps', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -313,11 +335,12 @@ jobs: env: RUSTFLAGS: "-A warnings" steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -333,36 +356,59 @@ jobs: && format('runs-on={0}/runner=amd64-ci-large/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-lint', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" - name: Install nightly for fmt run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rustfmt - - name: Rust Lint - Format - id: fmt - continue-on-error: true - run: cargo +$NIGHTLY_TOOLCHAIN fmt --all --check - - name: Rustc check - id: check - continue-on-error: true - run: RUSTFLAGS="-D warnings" cargo check --profile ci --locked --all-features --all-targets - - name: Rustc check (release) - id: check-release - continue-on-error: true - run: RUSTFLAGS="-D warnings" cargo check --locked --all-features --all-targets --release - - name: Rust Lint - Clippy All Features - id: clippy-all - continue-on-error: true - run: cargo clippy --profile ci --locked --all-features --all-targets -- -D warnings - - name: Rust Lint - Clippy Default Features - id: clippy-default - continue-on-error: true - run: cargo clippy --profile ci --locked --all-targets -- -D warnings + # Each Cargo invocation gets its own target directory to avoid output locks. sccache + # still shares compiler artifacts, while capping each build to a quarter of the host + # keeps the four compile-heavy checks from oversubscribing the runner. + - parallel: + - name: Rust Lint - Format + id: fmt + continue-on-error: true + run: cargo +$NIGHTLY_TOOLCHAIN fmt --all --check + - name: Rustc check (release) + id: check-release + continue-on-error: true + env: + CARGO_TARGET_DIR: ${{ runner.temp }}/vortex-lint/release + RUSTFLAGS: "-D warnings" + run: | + cargo check --jobs "$(( ($(nproc) + 3) / 4 ))" --locked \ + --all-features --all-targets --release + - name: Rust Lint - Clippy All Features + id: clippy-all + continue-on-error: true + env: + CARGO_TARGET_DIR: ${{ runner.temp }}/vortex-lint/clippy-all + run: | + cargo clippy --jobs "$(( ($(nproc) + 3) / 4 ))" --profile ci --locked \ + --all-features --all-targets -- -D warnings + - name: Rust Lint - Clippy Default Features + id: clippy-default + continue-on-error: true + env: + CARGO_TARGET_DIR: ${{ runner.temp }}/vortex-lint/clippy-default + run: | + cargo clippy --jobs "$(( ($(nproc) + 3) / 4 ))" --profile ci --locked \ + --all-targets -- -D warnings + - name: Rust Lint - Clippy No Default Features + id: clippy-no-default + continue-on-error: true + env: + CARGO_TARGET_DIR: ${{ runner.temp }}/vortex-lint/clippy-no-default + run: | + cargo hack --no-dev-deps --ignore-private clippy \ + --jobs "$(( ($(nproc) + 3) / 4 ))" --profile ci --no-default-features \ + -- -D warnings - name: Check lint results if: always() uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 @@ -370,10 +416,10 @@ jobs: script: | const failed = Object.entries({ fmt: '${{ steps.fmt.outcome }}', - check: '${{ steps.check.outcome }}', 'check-release': '${{ steps.check-release.outcome }}', 'clippy-all': '${{ steps.clippy-all.outcome }}', 'clippy-default': '${{ steps.clippy-default.outcome }}', + 'clippy-no-default': '${{ steps.clippy-no-default.outcome }}', }).filter(([, o]) => o === 'failure').map(([n]) => n); if (!failed.length) return; @@ -390,6 +436,10 @@ jobs: } core.setFailed(`Lint failed: ${failed.join(', ')}`); + - name: sccache stats + if: always() && github.repository == 'vortex-data/vortex' + run: sccache --show-stats + cpp-lint: name: "C/C++ (lint)" runs-on: ubuntu-latest @@ -405,27 +455,6 @@ jobs: | grep -v 'kernels/src/bit_unpack_.*_lanes\.cuh$' \ | xargs clang-format --dry-run --Werror --style=file - rust-lint-no-default: - name: "Rust (lint, no default)" - timeout-minutes: 30 - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-lint-no-default', github.run_id) - || 'ubuntu-latest' }} - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: ./.github/actions/setup-prebuild - with: - enable-sccache: "true" - - name: Rust Lint - Clippy No Default Features - shell: bash - run: | - cargo hack --no-dev-deps --ignore-private clippy --profile ci --no-default-features -- -D warnings - rust-test-windows: name: "Rust tests (windows-x64)" needs: duckdb-ready @@ -435,11 +464,12 @@ jobs: && format('runs-on={0}/pool=windows-x64-pre/extras=s3-cache', github.run_id) || 'windows-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Setup (Windows) run: | echo "C:\rust\cargo\bin" >> $env:GITHUB_PATH @@ -477,10 +507,11 @@ jobs: timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag=rust-test-linux-arm64 steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -507,11 +538,12 @@ jobs: && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=btrblocks-golden', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -539,17 +571,19 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 30 steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: ./.github/actions/setup-prebuild - with: - enable-sccache: "true" - - run: ./gradlew javadoc - working-directory: ./java - - run: ./gradlew check + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" + - uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6 + - name: Build documentation and run Java checks + run: ./gradlew --parallel javadoc check working-directory: ./java license-check-and-audit-check: @@ -577,11 +611,12 @@ jobs: && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-build', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -626,11 +661,12 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 30 steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -652,27 +688,29 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 30 steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" - - name: Build DuckDB tests - shell: bash - run: | - cargo nextest run --cargo-profile ci --locked -p vortex-duckdb --no-run - - name: Start MinIO - shell: bash - run: | - curl -sSL https://dl.min.io/server/minio/release/linux-amd64/minio -o "${RUNNER_TEMP}/minio" - chmod +x "${RUNNER_TEMP}/minio" - mkdir -p "${RUNNER_TEMP}/minio-data/vortex-test" - MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=minioadmin \ - "${RUNNER_TEMP}/minio" server "${RUNNER_TEMP}/minio-data" --address 127.0.0.1:9000 & - until curl -sf http://127.0.0.1:9000/minio/health/live; do sleep 1; done + - parallel: + - name: Build DuckDB tests + shell: bash + run: | + cargo nextest run --cargo-profile ci --locked -p vortex-duckdb --no-run + - name: Start MinIO + shell: bash + run: | + curl -sSL https://dl.min.io/server/minio/release/linux-amd64/minio -o "${RUNNER_TEMP}/minio" + chmod +x "${RUNNER_TEMP}/minio" + mkdir -p "${RUNNER_TEMP}/minio-data/vortex-test" + MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=minioadmin \ + "${RUNNER_TEMP}/minio" server "${RUNNER_TEMP}/minio-data" --address 127.0.0.1:9000 & + until curl -sf http://127.0.0.1:9000/minio/health/live; do sleep 1; done - name: Run tests shell: bash env: @@ -693,63 +731,72 @@ jobs: && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=wasm-integration', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: ./.github/actions/setup-prebuild - with: - enable-sccache: "true" - targets: "wasm32-wasip1" - - name: Setup Wasmer - shell: bash - run: | - curl https://get.wasmer.io -sSfL | sh - echo "$HOME/.wasmer/bin" >> $GITHUB_PATH + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" + targets: "wasm32-wasip1" + - name: Setup Wasmer + shell: bash + run: | + curl https://get.wasmer.io -sSfL | sh + echo "$HOME/.wasmer/bin" >> $GITHUB_PATH - run: cargo build --profile ci --target wasm32-wasip1 working-directory: ./wasm-test - run: wasmer run ./target/wasm32-wasip1/ci/wasm-test.wasm working-directory: ./wasm-test generated-files: - name: "Check generated source files are up to date" + name: "Generated source files and C API" runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=generated-files', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - # check-editions compares against the merge base, so it needs real history. - fetch-depth: 0 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # check-editions compares against the merge base, so it needs real history. + fetch-depth: 0 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" - name: Install nightly for cbindgen macro expansion run: rustup toolchain install $NIGHTLY_TOOLCHAIN - - name: "regenerate all .fbs/.proto Rust code" - run: | - cargo run --profile ci -p xtask -- generate-fbs - cargo run --profile ci -p xtask -- generate-proto - - name: "regenerate the edition records" - run: | - cargo run --profile ci -p xtask -- generate-editions + - parallel: + - name: "regenerate Rust sources and edition records" + env: + CARGO_TARGET_DIR: ${{ runner.temp }}/generated-files-target + run: | + cargo run --profile ci -p xtask -- generate-fbs + cargo run --profile ci -p xtask -- generate-proto + cargo run --profile ci -p xtask -- generate-editions + - name: "regenerate FFI header and test the C API" + run: | + cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi + cmake -S vortex-ffi -B "${RUNNER_TEMP}/ffi-c-build" -DRUST_BUILD_PROFILE=ci + cmake --build "${RUNNER_TEMP}/ffi-c-build" --parallel "$(nproc)" + ctest --test-dir "${RUNNER_TEMP}/ffi-c-build" --parallel "$(nproc)" - name: "check frozen edition records never change" # Independent of the regeneration above: a stale record must not mask a frozen one # being edited, nor the other way round. if: "!cancelled()" + env: + CARGO_TARGET_DIR: ${{ runner.temp }}/generated-files-target run: | BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}" cargo run --profile ci -p xtask -- check-editions --base "$BASE" - - name: "regenerate FFI header file" - run: | - cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi - name: "Make sure no files changed after regenerating" run: | git status --porcelain @@ -765,30 +812,3 @@ jobs: working-directory: vortex-flatbuffers/ run: | find flatbuffers/ -type f -name "*.fbs" | sed 's/^flatbuffers\///' | xargs -I{} -n1 flatc -I flatbuffers.HEAD --conform-includes flatbuffers --conform flatbuffers/{} flatbuffers.HEAD/{} - - ffi-c-test: - name: "C API test build" - timeout-minutes: 10 - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-build', github.run_id) - || 'ubuntu-latest' }} - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: ./.github/actions/setup-prebuild - with: - enable-sccache: "true" - - name: "regenerate FFI header file" - run: | - cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi - - name: Build and run C++ unit tests - run: | - cd vortex-ffi - mkdir build - cmake -Bbuild -DRUST_BUILD_PROFILE=ci - cmake --build build -j $(nproc) - ctest --test-dir build -j $(nproc) diff --git a/.github/workflows/compat-validation.yml b/.github/workflows/compat-validation.yml index d8012193614..3e8746950a6 100644 --- a/.github/workflows/compat-validation.yml +++ b/.github/workflows/compat-validation.yml @@ -29,11 +29,12 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 30 steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} diff --git a/.github/workflows/cuda.yaml b/.github/workflows/cuda.yaml index 87bb4887790..a8124c9dbbd 100644 --- a/.github/workflows/cuda.yaml +++ b/.github/workflows/cuda.yaml @@ -56,10 +56,11 @@ jobs: timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=gpu/extras=s3-cache/tag=cuda-build steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -91,23 +92,25 @@ jobs: timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=gpu/extras=s3-cache/tag=cuda-tests steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "true" - - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 - with: - sync: false + - parallel: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: "true" + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 + with: + sync: false - name: Install nextest uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2 with: @@ -161,15 +164,16 @@ jobs: - sanitizer: initcheck runner_flags: "--tool initcheck --error-exitcode 1" steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -190,15 +194,16 @@ jobs: timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=gpu/extras=s3-cache/tag=cuda-test-cudf steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish-dry-runs.yml b/.github/workflows/publish-dry-runs.yml index bb10fc9fc82..45a1210a6ff 100644 --- a/.github/workflows/publish-dry-runs.yml +++ b/.github/workflows/publish-dry-runs.yml @@ -74,21 +74,23 @@ jobs: runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag=check-java-publish-build-arm64 runs-on: ${{ matrix.runner }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-depth: 0 - - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 - with: - distribution: "corretto" - java-version: "17" - - uses: ./.github/actions/setup-prebuild - with: - enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + - parallel: + - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 + with: + distribution: "corretto" + java-version: "17" + - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} + - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 - name: Install cargo-zigbuild uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 with: @@ -109,11 +111,12 @@ jobs: && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-publish-dry-run', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} diff --git a/.github/workflows/rust-instrumented.yml b/.github/workflows/rust-instrumented.yml index d186ce8dbac..9bfab357916 100644 --- a/.github/workflows/rust-instrumented.yml +++ b/.github/workflows/rust-instrumented.yml @@ -61,11 +61,12 @@ jobs: LLVM_PROFILE_FILE: "target/coverage/vortex-%p-%m.profraw" GRCOV_OUTPUT_FILE: "target/coverage/vortex.lcov" steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -126,11 +127,12 @@ jobs: && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-coverage', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -184,11 +186,12 @@ jobs: # unset -Zsanitizer flag and we should allow that. RUSTFLAGS: "-A warnings -Cunsafe-allow-abi-mismatch=sanitizer -C debuginfo=2 -C opt-level=0 -C strip=none" steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -253,11 +256,12 @@ jobs: && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-ffi-test-sanitizer', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -305,11 +309,12 @@ jobs: RUSTFLAGS: "-A warnings" RUST_BACKTRACE: full steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" From a53eae54975d36c78f50459b2e62a47bd014ed77 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Thu, 3 Sep 2026 21:47:46 +0100 Subject: [PATCH 2/2] Avoid competing Rust builds in parallel steps Signed-off-by: Robert Kruszewski --- .github/workflows/ci.yml | 157 +++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 90 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8be6fdae7a2..0cdb87ba7f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,19 +120,18 @@ jobs: uv run --all-packages pytest --benchmark-disable -n auto test/ working-directory: vortex-python/ - - parallel: - # The datasets integration relies on private `datasets` APIs, so exercise the oldest - # supported release in addition to the locked one. - - name: Pytest - Vortex (datasets 4.x floor) - run: | - uv run --all-packages --with 'datasets==4.0.0' --with 'huggingface-hub<1' \ - pytest --benchmark-disable test/test_hf_datasets.py - working-directory: vortex-python/ + # The datasets integration relies on private `datasets` APIs, so exercise the oldest + # supported release in addition to the locked one. + - name: Pytest - Vortex (datasets 4.x floor) + run: | + uv run --all-packages --with 'datasets==4.0.0' --with 'huggingface-hub<1' \ + pytest --benchmark-disable test/test_hf_datasets.py + working-directory: vortex-python/ - - name: Check Python documentation - run: | - uv run --all-packages make check - working-directory: docs/ + - name: Check Python documentation + run: | + uv run --all-packages make check + working-directory: docs/ # Keep this last: setup-benchmark.sh confines everything except the wrapped # benchmark to the housekeeping CPUs, so any step after it runs on few cores. @@ -254,8 +253,12 @@ jobs: fail-fast: false matrix: config: - - name: "native feature sets" + - name: "all-features" runner: amd64-ci-large + args: "--all-features --all-targets" + - name: "default features" + runner: amd64-ci-large + args: "--all-targets" - name: "with tokio dispatcher" runner: amd64-ci-small # Only build the crates that have the tokio features, not re-building other crates with no-default-features @@ -277,25 +280,7 @@ jobs: with: enable-sccache: "true" targets: ${{ matrix.config.target }} - - parallel: - - name: Build all features - if: matrix.config.name == 'native feature sets' - uses: ./.github/actions/check-rebuild - with: - command: >- - CARGO_TARGET_DIR=${{ runner.temp }}/vortex-build/all-features - cargo hack build --jobs "$(( ($(nproc) + 1) / 2 ))" --profile ci --locked - --all-features --all-targets --ignore-private - - name: Build default features - if: matrix.config.name == 'native feature sets' - uses: ./.github/actions/check-rebuild - with: - command: >- - CARGO_TARGET_DIR=${{ runner.temp }}/vortex-build/default-features - cargo hack build --jobs "$(( ($(nproc) + 1) / 2 ))" --profile ci --locked - --all-targets --ignore-private - uses: ./.github/actions/check-rebuild - if: matrix.config.name != 'native feature sets' with: command: "${{matrix.config.env.rustflags}} cargo hack build --profile ci --locked ${{matrix.config.args}} --ignore-private" - name: "Make sure no files changed after build" @@ -367,48 +352,24 @@ jobs: enable-sccache: "true" - name: Install nightly for fmt run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rustfmt - # Each Cargo invocation gets its own target directory to avoid output locks. sccache - # still shares compiler artifacts, while capping each build to a quarter of the host - # keeps the four compile-heavy checks from oversubscribing the runner. - - parallel: - - name: Rust Lint - Format - id: fmt - continue-on-error: true - run: cargo +$NIGHTLY_TOOLCHAIN fmt --all --check - - name: Rustc check (release) - id: check-release - continue-on-error: true - env: - CARGO_TARGET_DIR: ${{ runner.temp }}/vortex-lint/release - RUSTFLAGS: "-D warnings" - run: | - cargo check --jobs "$(( ($(nproc) + 3) / 4 ))" --locked \ - --all-features --all-targets --release - - name: Rust Lint - Clippy All Features - id: clippy-all - continue-on-error: true - env: - CARGO_TARGET_DIR: ${{ runner.temp }}/vortex-lint/clippy-all - run: | - cargo clippy --jobs "$(( ($(nproc) + 3) / 4 ))" --profile ci --locked \ - --all-features --all-targets -- -D warnings - - name: Rust Lint - Clippy Default Features - id: clippy-default - continue-on-error: true - env: - CARGO_TARGET_DIR: ${{ runner.temp }}/vortex-lint/clippy-default - run: | - cargo clippy --jobs "$(( ($(nproc) + 3) / 4 ))" --profile ci --locked \ - --all-targets -- -D warnings - - name: Rust Lint - Clippy No Default Features - id: clippy-no-default - continue-on-error: true - env: - CARGO_TARGET_DIR: ${{ runner.temp }}/vortex-lint/clippy-no-default - run: | - cargo hack --no-dev-deps --ignore-private clippy \ - --jobs "$(( ($(nproc) + 3) / 4 ))" --profile ci --no-default-features \ - -- -D warnings + # Run one Cargo command at a time so it can use the whole runner and reuse + # artifacts from the shared target directory in subsequent checks. + - name: Rust Lint - Format + id: fmt + continue-on-error: true + run: cargo +$NIGHTLY_TOOLCHAIN fmt --all --check + - name: Rustc check (release) + id: check-release + continue-on-error: true + run: RUSTFLAGS="-D warnings" cargo check --locked --all-features --all-targets --release + - name: Rust Lint - Clippy All Features + id: clippy-all + continue-on-error: true + run: cargo clippy --profile ci --locked --all-features --all-targets -- -D warnings + - name: Rust Lint - Clippy Default Features + id: clippy-default + continue-on-error: true + run: cargo clippy --profile ci --locked --all-targets -- -D warnings - name: Check lint results if: always() uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 @@ -419,7 +380,6 @@ jobs: 'check-release': '${{ steps.check-release.outcome }}', 'clippy-all': '${{ steps.clippy-all.outcome }}', 'clippy-default': '${{ steps.clippy-default.outcome }}', - 'clippy-no-default': '${{ steps.clippy-no-default.outcome }}', }).filter(([, o]) => o === 'failure').map(([n]) => n); if (!failed.length) return; @@ -440,6 +400,28 @@ jobs: if: always() && github.repository == 'vortex-data/vortex' run: sccache --show-stats + rust-lint-no-default: + name: "Rust (lint, no default)" + timeout-minutes: 30 + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-lint-no-default', github.run_id) + || 'ubuntu-latest' }} + steps: + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" + - name: Rust Lint - Clippy No Default Features + shell: bash + run: | + cargo hack --no-dev-deps --ignore-private clippy --profile ci --no-default-features -- -D warnings + cpp-lint: name: "C/C++ (lint)" runs-on: ubuntu-latest @@ -774,29 +756,24 @@ jobs: enable-sccache: "true" - name: Install nightly for cbindgen macro expansion run: rustup toolchain install $NIGHTLY_TOOLCHAIN - - parallel: - - name: "regenerate Rust sources and edition records" - env: - CARGO_TARGET_DIR: ${{ runner.temp }}/generated-files-target - run: | - cargo run --profile ci -p xtask -- generate-fbs - cargo run --profile ci -p xtask -- generate-proto - cargo run --profile ci -p xtask -- generate-editions - - name: "regenerate FFI header and test the C API" - run: | - cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi - cmake -S vortex-ffi -B "${RUNNER_TEMP}/ffi-c-build" -DRUST_BUILD_PROFILE=ci - cmake --build "${RUNNER_TEMP}/ffi-c-build" --parallel "$(nproc)" - ctest --test-dir "${RUNNER_TEMP}/ffi-c-build" --parallel "$(nproc)" + - name: "regenerate Rust sources and edition records" + run: | + cargo run --profile ci -p xtask -- generate-fbs + cargo run --profile ci -p xtask -- generate-proto + cargo run --profile ci -p xtask -- generate-editions - name: "check frozen edition records never change" # Independent of the regeneration above: a stale record must not mask a frozen one # being edited, nor the other way round. if: "!cancelled()" - env: - CARGO_TARGET_DIR: ${{ runner.temp }}/generated-files-target run: | BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}" cargo run --profile ci -p xtask -- check-editions --base "$BASE" + - name: "regenerate FFI header and test the C API" + run: | + cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi + cmake -S vortex-ffi -B "${RUNNER_TEMP}/ffi-c-build" -DRUST_BUILD_PROFILE=ci + cmake --build "${RUNNER_TEMP}/ffi-c-build" --parallel "$(nproc)" + ctest --test-dir "${RUNNER_TEMP}/ffi-c-build" --parallel "$(nproc)" - name: "Make sure no files changed after regenerating" run: | git status --porcelain