diff --git a/.github/workflows/docker-build-check.yml b/.github/workflows/docker-build-check.yml index dea57269c..34da220fc 100644 --- a/.github/workflows/docker-build-check.yml +++ b/.github/workflows/docker-build-check.yml @@ -4,11 +4,24 @@ name: Docker Build Check +# Not a required status check, so a workflow-level filter is safe here: a pull +# request that skips it is not left waiting on a check that never arrives. What +# it verifies -- that the builder images still build and that their pinned +# package lists still match -- cannot change unless something under `dstack/` +# does. on: push: branches: [ next, 'release/**' ] + paths: + - 'dstack/**' + - 'rust-toolchain.toml' + - '.github/workflows/docker-build-check.yml' pull_request: branches: [ next, 'release/**' ] + paths: + - 'dstack/**' + - 'rust-toolchain.toml' + - '.github/workflows/docker-build-check.yml' env: DSTACK_REV: ${{ github.event.pull_request.head.sha || github.sha }} @@ -36,6 +49,15 @@ jobs: build-args: | DSTACK_REV=${{ env.DSTACK_REV }} DSTACK_SRC_URL=${{ env.DSTACK_SRC_URL }} + # Layer cache, scoped per image so the three jobs do not overwrite one + # another's. It covers the pinned-package install and the toolchain + # setup. The cargo build below cannot be cached: the source arrives + # through `git clone` at DSTACK_REV inside the build rather than from + # the build context, so BuildKit has nothing to key it on. That is + # deliberate -- the revision is what the image records in + # /etc/.GIT_REV -- so the ceiling here is the layers above the clone. + cache-from: type=gha,scope=gateway + cache-to: type=gha,mode=max,scope=gateway - name: Verify pinned packages run: | @@ -87,6 +109,15 @@ jobs: build-args: | DSTACK_REV=${{ env.DSTACK_REV }} DSTACK_SRC_URL=${{ env.DSTACK_SRC_URL }} + # Layer cache, scoped per image so the three jobs do not overwrite one + # another's. It covers the pinned-package install and the toolchain + # setup. The cargo build below cannot be cached: the source arrives + # through `git clone` at DSTACK_REV inside the build rather than from + # the build context, so BuildKit has nothing to key it on. That is + # deliberate -- the revision is what the image records in + # /etc/.GIT_REV -- so the ceiling here is the layers above the clone. + cache-from: type=gha,scope=kms + cache-to: type=gha,mode=max,scope=kms - name: Verify pinned packages run: | @@ -140,6 +171,15 @@ jobs: build-args: | DSTACK_REV=${{ env.DSTACK_REV }} DSTACK_SRC_URL=${{ env.DSTACK_SRC_URL }} + # Layer cache, scoped per image so the three jobs do not overwrite one + # another's. It covers the pinned-package install and the toolchain + # setup. The cargo build below cannot be cached: the source arrives + # through `git clone` at DSTACK_REV inside the build rather than from + # the build context, so BuildKit has nothing to key it on. That is + # deliberate -- the revision is what the image records in + # /etc/.GIT_REV -- so the ceiling here is the layers above the clone. + cache-from: type=gha,scope=verifier + cache-to: type=gha,mode=max,scope=verifier - name: Verify pinned packages (runtime) run: | diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index f028ea67e..0cb18deec 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -21,17 +21,63 @@ jobs: working-directory: dstack steps: - uses: actions/checkout@v5 + with: + # The change detection below diffs against the base commit. + fetch-depth: 0 + + # Which files changed, decided here rather than with a workflow-level + # `paths:` filter. + # + # `rust-checks` is a required status check. A workflow skipped by `paths:` + # reports nothing at all, so the check never arrives and the pull request + # waits on it forever -- the filter has to live inside a job that always + # runs and always reports. Everything expensive below is gated on this + # output; when nothing relevant moved the job costs a checkout and a diff. + - name: Detect relevant changes + id: changes + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + if [ -z "$BASE_SHA" ]; then + echo "relevant=true" >> "$GITHUB_OUTPUT" + echo "push build: running everything" + exit 0 + fi + changed=$(git diff --name-only "$BASE_SHA...HEAD") + echo "$changed" + if echo "$changed" | grep -qE '^(dstack/|sdk/simulator/|rust-toolchain\.toml|\.github/workflows/rust\.yml)'; then + echo "relevant=true" >> "$GITHUB_OUTPUT" + else + echo "relevant=false" >> "$GITHUB_OUTPUT" + echo "nothing this job covers changed" + fi - name: Install Rust uses: dtolnay/rust-toolchain@1.92.0 with: components: clippy, rustfmt + # The largest job in the repository was compiling the workspace from nothing + # on every run. `simulator-release.yml` already caches this way; this brings + # the check that gates every merge in line with it. + # + # `workspaces` because the manifest lives in `dstack/`, not at the root, and + # the action keys the cache on the lockfile it finds there. `sdk/` is a + # separate workspace, cached by its own job. + - name: Cache cargo + if: steps.changes.outputs.relevant == 'true' + uses: Swatinem/rust-cache@v2 + with: + workspaces: dstack + - name: Run Clippy + if: steps.changes.outputs.relevant == 'true' run: cargo clippy -- -D warnings -D clippy::expect_used -D clippy::unwrap_used --allow unused_variables - name: Cargo fmt check + if: steps.changes.outputs.relevant == 'true' run: cargo fmt --check --all - name: Run tests + if: steps.changes.outputs.relevant == 'true' run: ./run-tests.sh diff --git a/.github/workflows/sdk-compat.yaml b/.github/workflows/sdk-compat.yaml index e4e4efe2d..2fdc127a6 100644 --- a/.github/workflows/sdk-compat.yaml +++ b/.github/workflows/sdk-compat.yaml @@ -7,11 +7,29 @@ name: SDK compatibility permissions: contents: read +# Released SDKs against the current agent: nothing to re-check unless the SDKs +# or the agent they talk to changed. Not a required status check, so a +# workflow-level filter is safe. +# +# `dstack/**` rather than the guest-agent directories alone. The agent under +# test is built through `simulator_start`, whose dependency closure reaches a +# dozen crates in that workspace; naming the three obvious ones would leave the +# rest silently uncovered the moment one of them changed behaviour. on: push: branches: [next, 'release/**'] + paths: + - 'sdk/**' + - 'dstack/**' + - 'rust-toolchain.toml' + - '.github/workflows/sdk-compat.yaml' pull_request: branches: [next, 'release/**'] + paths: + - 'sdk/**' + - 'dstack/**' + - 'rust-toolchain.toml' + - '.github/workflows/sdk-compat.yaml' env: CARGO_TERM_COLOR: always @@ -24,8 +42,40 @@ env: RUSTUP_TOOLCHAIN: 1.92.0 jobs: + # The agent the released SDKs are tested against, built once. + # + # Both matrix legs call `simulator_start`, which builds + # `dstack-guest-agent-simulator` out of the current tree -- the same binary, + # from the same commit, compiled twice in parallel. Nothing here replaces that + # call or hands the legs a binary from elsewhere: they still build from their + # own checkout, so what they test is still what this commit produces. This job + # only puts the artifacts in the cache first, under a key both legs restore, + # so the build they run finds its work already done. + # + # A `needs:` rather than a skip switch in `simulator_build` on purpose. That + # switch would be a path where the binary under test did not come from the + # checkout, which is not worth trading for a few minutes. + simulator: + name: Build the agent under test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install Rust + uses: dtolnay/rust-toolchain@1.92.0 + + - name: Cache cargo + uses: Swatinem/rust-cache@v2 + with: + workspaces: dstack + shared-key: sdk-compat-agent + + - name: Build the simulator + run: ./sdk/simulator/build.sh + sdk-compat: name: ${{ matrix.tag }} SDKs vs current agent + needs: simulator runs-on: ubuntu-latest strategy: # Each tag is an independent claim; one failing should not hide the other. @@ -62,6 +112,16 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@1.92.0 + # Restore only. The `simulator` job above populated this key; a leg that + # saved as well would race its twin for the same entry and gain nothing, + # since the next run's `simulator` job writes it again anyway. + - name: Restore the agent build + uses: Swatinem/rust-cache@v2 + with: + workspaces: dstack + shared-key: sdk-compat-agent + save-if: false + - name: Install Go uses: actions/setup-go@v5 with: diff --git a/.github/workflows/sdk.yaml b/.github/workflows/sdk.yaml index 906192202..88362d5f5 100644 --- a/.github/workflows/sdk.yaml +++ b/.github/workflows/sdk.yaml @@ -21,6 +21,44 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 + with: + # The change detection below diffs against the base commit. + fetch-depth: 0 + + # Which files changed, decided here rather than with a workflow-level + # `paths:` filter. + # + # `sdk-tests` is a required status check. A workflow skipped by `paths:` + # reports nothing at all, so the check never arrives and the pull request + # waits on it forever -- the filter has to live inside a job that always + # runs and always reports. Everything expensive below is gated on this + # output; when nothing relevant moved the job costs a checkout and a diff. + # + # `dstack/` is in the pattern because `sdk/run-tests.sh` starts the + # simulator, which is built from that workspace -- these suites exercise + # the agent's wire surface, not just the client libraries. The whole + # directory rather than the simulator's dependency closure: the closure is + # a dozen crates deep and would go stale the first time a dependency + # moved, and a filter that silently stops covering something is worse than + # one that occasionally runs when it need not. + - name: Detect relevant changes + id: changes + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + if [ -z "$BASE_SHA" ]; then + echo "relevant=true" >> "$GITHUB_OUTPUT" + echo "push build: running everything" + exit 0 + fi + changed=$(git diff --name-only "$BASE_SHA...HEAD") + echo "$changed" + if echo "$changed" | grep -qE '^(sdk/|dstack/|rust-toolchain\.toml|\.github/workflows/sdk\.yaml)'; then + echo "relevant=true" >> "$GITHUB_OUTPUT" + else + echo "relevant=false" >> "$GITHUB_OUTPUT" + echo "nothing this job covers changed" + fi - name: Install Rust uses: dtolnay/rust-toolchain@1.92.0 @@ -29,14 +67,23 @@ jobs: # This additional target is needed for wasm32 compatibility check. targets: wasm32-unknown-unknown, thumbv6m-none-eabi + - name: Cache cargo + if: steps.changes.outputs.relevant == 'true' + uses: Swatinem/rust-cache@v2 + with: + workspaces: sdk/rust + - name: SDK tests + if: steps.changes.outputs.relevant == 'true' run: cd sdk && ./run-tests.sh - name: Verify WASM compilation + if: steps.changes.outputs.relevant == 'true' # Ensures SDK types can be used in smart contracts run: cargo check --manifest-path sdk/rust/Cargo.toml --target=wasm32-unknown-unknown -p dstack-sdk-types - name: Verify no_std compatibility + if: steps.changes.outputs.relevant == 'true' run: | cargo test --manifest-path sdk/rust/Cargo.toml -p dstack-sdk-types --test no_std_test --no-default-features cargo check --manifest-path sdk/rust/Cargo.toml -p no_std_check --target thumbv6m-none-eabi diff --git a/.github/workflows/vmm-ui.yml b/.github/workflows/vmm-ui.yml index 2abda43db..259865541 100644 --- a/.github/workflows/vmm-ui.yml +++ b/.github/workflows/vmm-ui.yml @@ -7,11 +7,19 @@ name: VMM UI build permissions: contents: read +# Only the VMM's web UI is built here. Not a required status check, so a +# workflow-level filter is safe. on: push: branches: [ next, 'release/**' ] + paths: + - 'dstack/vmm/ui/**' + - '.github/workflows/vmm-ui.yml' pull_request: branches: [ next, 'release/**' ] + paths: + - 'dstack/vmm/ui/**' + - '.github/workflows/vmm-ui.yml' jobs: build: