diff --git a/.agent/CODING_TASTE.md b/.agent/CODING_TASTE.md index 65c93260e..f29a14bcf 100644 --- a/.agent/CODING_TASTE.md +++ b/.agent/CODING_TASTE.md @@ -25,7 +25,7 @@ quantified and bounded. parallel paths (a separate `GetAppKeyAmd` was rejected on these grounds, #630). When an existing API is the wrong shape, add a purpose-built one rather than overloading a return value (`is_app_allowed` returning policy → add `auth_api.get_app_policy` instead, #538). -- **Names must say what the thing does.** `GetQuote` for an app key → `GetAttestationForAppKey` +- **Names must say what the thing does.** `GetQuote` for an app key → `AttestAppKey` (#360). An RPC named `ComposeHash` that returns an `app_id` is wrong (#181). - **Avoid enums in protobuf APIs** that surface as JSON — proto has no way to express snake_case serde renaming, so use strings (#241). @@ -257,6 +257,11 @@ not just the symptom: ## Readability and idioms +- **Import for readability, not minimum path length.** Import specific types and + frequently repeated operations when their short names remain unambiguous + (`TcpListener::bind`, `timeout`, `Command::new`). Keep qualifiers that carry useful + semantic context (`serde_json::from_slice`, `tokio::spawn`, `anyhow::bail!`). Follow + the surrounding module when either form is equally clear. - **Errors**: `anyhow` everywhere in services — `thiserror` only in library crates whose callers match on error variants. `bail!` inside an `if` is the guard idiom; **`ensure!` is never used in this codebase** (0 occurrences) and reads as foreign. `let ... else diff --git a/.claude/agents/sdk-sync-checker.md b/.claude/agents/sdk-sync-checker.md index 794123da8..4e4050109 100644 --- a/.claude/agents/sdk-sync-checker.md +++ b/.claude/agents/sdk-sync-checker.md @@ -67,10 +67,10 @@ Details: ## Locations - Protos: `dstack/guest-agent/rpc/proto/*.proto` -- Python: `sdk/python/src/dstack_sdk/dstack_client.py` -- Go: `sdk/go/dstack/client.go` -- Rust: `sdk/rust/types/src/dstack.rs` -- JS: `sdk/js/src/index.ts` +- Python: `sdk/python/src/dstack_sdk/dstack_client_v0.py`, `dstack_client_v1.py` +- Go: `sdk/go/dstack/client_v0.go`, `client_v1.go` +- Rust: `sdk/rust/types/src/dstack_v0.rs`, `dstack_v1.rs` +- JS: `sdk/js/src/client-v0.ts`, `client-v1.ts` - Docs: `sdk/curl/api.md`, `sdk/curl/api-tappd.md` Focus on API surface differences. Provide specific file paths and line numbers. diff --git a/.github/workflows/docker-build-check.yml b/.github/workflows/docker-build-check.yml index 25dcfc677..318df35a8 100644 --- a/.github/workflows/docker-build-check.yml +++ b/.github/workflows/docker-build-check.yml @@ -4,17 +4,57 @@ name: Docker Build Check +# Not a required status check, so a workflow-level filter is safe here: a pull +# request that skips it is not left waiting on a check that never arrives. What +# it verifies -- that the builder images still build and that their pinned +# package lists still match -- cannot change unless something under `dstack/` +# does. on: push: - branches: [ master, next, dev-* ] + branches: [ next, 'release/**' ] + paths: + - 'dstack/**' + - 'rust-toolchain.toml' + - '.github/workflows/docker-build-check.yml' + - '.github/workflows/kms-release.yml' + - '.github/workflows/gateway-release.yml' + - '.github/workflows/verifier-release.yml' pull_request: - branches: [ master, next, dev-* ] + branches: [ next, 'release/**' ] + paths: + - 'dstack/**' + - 'rust-toolchain.toml' + - '.github/workflows/docker-build-check.yml' + - '.github/workflows/kms-release.yml' + - '.github/workflows/gateway-release.yml' + - '.github/workflows/verifier-release.yml' env: DSTACK_REV: ${{ github.event.pull_request.head.sha || github.sha }} DSTACK_SRC_URL: ${{ github.event.pull_request.head.repo.clone_url || format('{0}/{1}', github.server_url, github.repository) }} +# Builds go through the `docker` driver, i.e. the daemon's own BuildKit, rather +# than the `docker-container` builder setup-buildx creates by default. With a +# container builder every `--load` compresses the result into a tarball and +# imports it into the daemon; for the `*-builder` targets, which carry the Rust +# toolchain and the cargo target directory, that alone was two to three +# minutes. With the daemon's builder the image lands in the store directly, and +# the second build of the `*-builder` target is a cache hit on the first. +# +# No BuildKit layer cache (`cache-to: type=gha`). The source is cloned at +# DSTACK_REV inside the build, so the cargo layer misses on every commit, and +# `mode=max` still exported the whole builder stage to the Actions cache: about +# two and a half minutes per job, and most of the repository's 10 GB cache +# quota, which evicted the cargo caches the other workflows depend on. What it +# could have saved was the pinned-package install, roughly fifteen seconds. jobs: + build-script-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - name: Test release build orchestration + run: python3 -m unittest discover -s dstack/build/shared/tests -v + gateway: runs-on: ubuntu-latest steps: @@ -22,6 +62,8 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + driver: docker - name: Build Gateway Docker image uses: docker/build-push-action@v5 @@ -73,6 +115,8 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + driver: docker - name: Build KMS Docker image uses: docker/build-push-action@v5 @@ -88,10 +132,10 @@ jobs: DSTACK_REV=${{ env.DSTACK_REV }} DSTACK_SRC_URL=${{ env.DSTACK_SRC_URL }} - - name: Verify pinned packages (qemu stage) + - name: Verify pinned packages run: | dstack/build/shared/verify-pinned-packages.sh dstack-kms-check:latest \ - dstack/kms/dstack-app/builder/shared/qemu-pinned-packages.txt + dstack/kms/dstack-app/builder/shared/pinned-packages.txt - name: Build kms-builder target run: | @@ -125,6 +169,8 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + driver: docker - name: Build Verifier Docker image uses: docker/build-push-action@v5 @@ -163,21 +209,3 @@ jobs: run: | dstack/build/shared/verify-pinned-packages.sh verifier-builder-check:latest \ dstack/verifier/builder/shared/builder-pinned-packages.txt - - - name: Build acpi-builder target - run: | - docker buildx build \ - --load \ - --target acpi-builder \ - --tag verifier-acpi-check:latest \ - --provenance=false \ - --file dstack/verifier/builder/Dockerfile \ - --build-context build-shared=dstack/build/shared \ - --build-arg "DSTACK_REV=${DSTACK_REV}" \ - --build-arg "DSTACK_SRC_URL=${DSTACK_SRC_URL}" \ - dstack/verifier - - - name: Verify qemu pinned packages - run: | - dstack/build/shared/verify-pinned-packages.sh verifier-acpi-check:latest \ - dstack/verifier/builder/shared/qemu-pinned-packages.txt diff --git a/.github/workflows/foundry-test.yml b/.github/workflows/foundry-test.yml index 282d7056b..715431aad 100644 --- a/.github/workflows/foundry-test.yml +++ b/.github/workflows/foundry-test.yml @@ -56,6 +56,9 @@ jobs: forge build --sizes id: build + - name: Install OpenZeppelin upgrades validator + run: npm install --no-save --package-lock=false @openzeppelin/upgrades-core@^1.37.0 + - name: Run Forge tests run: | forge test --ffi -vvv diff --git a/.github/workflows/gateway-cluster-tests.yml b/.github/workflows/gateway-cluster-tests.yml new file mode 100644 index 000000000..74acca7f3 --- /dev/null +++ b/.github/workflows/gateway-cluster-tests.yml @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +name: Gateway cluster tests + +# WaveKV replication between gateway nodes: push and periodic sync, anti-entropy +# repair, bootstrap after losing a store, partition recovery, node identity, and +# the admin RPCs that gate registration. Unit tests cover the store; this runs +# three real gateways and stops, wipes and restarts them. +# +# The nodes authenticate each other for real, so the cluster mTLS path is +# exercised rather than switched off -- which is what the process-based suite +# this replaces could not do. +on: + push: + branches: [ next, 'release/**' ] + paths: + - 'dstack/gateway/**' + - 'dstack/cert-client/**' + - 'dstack/ra-tls/**' + - 'dstack/guest-agent-simulator/**' + - 'sdk/simulator/**' + - 'dstack/crates/mock-attestation/**' + - '.github/workflows/gateway-cluster-tests.yml' + pull_request: + branches: [ next, 'release/**' ] + paths: + - 'dstack/gateway/**' + - 'dstack/cert-client/**' + - 'dstack/ra-tls/**' + - 'dstack/guest-agent-simulator/**' + - 'sdk/simulator/**' + - 'dstack/crates/mock-attestation/**' + - '.github/workflows/gateway-cluster-tests.yml' + +# Per workflow, not shared with the other two gateway suites. +# +# A group shared across all three does not serialise them, it drops one: a +# concurrency group holds a single *pending* run, so when the third workflow +# queued behind the first two, the one already waiting was cancelled -- +# silently, and reported as a cancelled run rather than a failure. The suites +# no longer need serialising anyway; each brings up its own attestation fixture +# under its own `FIXTURE_NS`, so there is nothing left to collide over. +# +# `cancel-in-progress` is the ordinary meaning here: a new push supersedes the +# run for the commit it replaced. +concurrency: + group: gateway-cluster-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + +jobs: + gateway-cluster: + runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} + # 28 tests, most of which restart nodes and then wait out a 5s sync interval. + timeout-minutes: 60 + steps: + - uses: actions/checkout@v5 + + - name: Install Rust + uses: dtolnay/rust-toolchain@1.92.0 + with: + targets: x86_64-unknown-linux-musl + + - name: Install musl toolchain + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends musl-tools + + # The gateways create real WireGuard interfaces inside their containers. + - name: Record kernel capabilities + run: | + echo "kernel: $(uname -r)" + sudo modprobe wireguard 2>&1 || echo "no wireguard module available" + echo "wireguard loaded: $(lsmod | grep -c '^wireguard ' || true)" + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + dstack/target + key: gateway-cluster-${{ runner.os }}-${{ hashFiles('dstack/Cargo.lock') }} + restore-keys: gateway-cluster-${{ runner.os }}- + + - name: Cluster suite + working-directory: dstack/gateway/test-run/cluster + run: ./run-cluster-tests.sh + + # The suite only materialises node logs for the two tests that assert on + # them, so the mounted directory is close to empty for any other failure. + # Pull them from the daemon instead, by compose service so the shared + # fixture's project-scoped names do not have to be guessed. + - name: Collect node logs on failure + if: failure() + working-directory: dstack/gateway/test-run/cluster + # The compose files require `${FIXTURE_NS:?}`, and the suite script + # exports it only to itself. Unset here, every `docker compose` call in + # this step -- not just the fixture one -- fails at interpolation, and + # the step still passes, so the artifact carries no container logs. + env: + FIXTURE_NS: dstack-fixture-cluster + run: | + mkdir -p run/logs + # By project, discovered from the daemon. A bare `docker compose` here + # addresses the project named after this directory (`cluster`), which + # no test ever uses -- every test runs under `cluster-`, so + # the loop that used `compose config --services` collected nothing. + docker ps -a --filter 'label=com.docker.compose.project' \ + --format '{{.Label "com.docker.compose.project"}}' \ + | grep -E '^cluster-' | sort -u | while read -r project; do + docker compose -p "$project" -f docker-compose.yml logs --no-color \ + > "run/logs/project-$project.log" 2>&1 || true + done + # The suite's own per-test dumps are root-owned inside the bind mount. + docker run --rm -v "$PWD/run:/r" alpine:latest chmod -R a+rX /r || true + # The attestation fixture is a project of its own now, so it is not in + # this suite's service list and its logs have to be asked for + # separately -- they are where a quote-verification failure explains + # itself. + docker compose -p "$FIXTURE_NS" -f ../attestation/fixture.yml logs \ + --no-color > "run/logs/fixture.log" 2>&1 || true + docker compose ps -a > run/logs/compose-ps.txt 2>&1 || true + + - name: Upload node logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: gateway-cluster-logs + path: dstack/gateway/test-run/cluster/run/logs/ + if-no-files-found: ignore + retention-days: 7 + + - name: Tear down + if: always() + working-directory: dstack/gateway/test-run/cluster + run: ./run-cluster-tests.sh down diff --git a/.github/workflows/gateway-e2e-tests.yml b/.github/workflows/gateway-e2e-tests.yml new file mode 100644 index 000000000..38aa838fe --- /dev/null +++ b/.github/workflows/gateway-e2e-tests.yml @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +name: Gateway e2e tests + +# The gateway's certbot/ACME half only exists as a whole: certificate issuance, +# cluster sync of the result, and the dns-persist-01 flow each depend on the +# ones before. Unit tests cover the pieces; this stands up three gateways, a +# Pebble CA, a mock Cloudflare DNS API and a mock attestation collateral +# service, and asserts on what the cluster actually converges to. +# +# The suite verifies peer quotes for real: the gateways check each other against +# development trust anchors derived from the simulator's seed, so it covers the +# cluster mTLS path that a harness with the checks switched off cannot reach. +on: + push: + branches: [ next, 'release/**' ] + paths: + - 'dstack/gateway/**' + - 'dstack/cert-client/**' + - 'dstack/certbot/**' + - 'dstack/ra-tls/**' + - 'dstack/guest-agent-simulator/**' + - 'sdk/simulator/**' + - 'dstack/crates/mock-attestation/**' + - 'tools/mock-cf-dns/**' + - '.github/workflows/gateway-e2e-tests.yml' + pull_request: + branches: [ next, 'release/**' ] + paths: + - 'dstack/gateway/**' + - 'dstack/cert-client/**' + - 'dstack/certbot/**' + - 'dstack/ra-tls/**' + - 'dstack/guest-agent-simulator/**' + - 'sdk/simulator/**' + - 'dstack/crates/mock-attestation/**' + - 'tools/mock-cf-dns/**' + - '.github/workflows/gateway-e2e-tests.yml' + +# Per workflow, not shared with the other two gateway suites. +# +# A group shared across all three does not serialise them, it drops one: a +# concurrency group holds a single *pending* run, so when the third workflow +# queued behind the first two, the one already waiting was cancelled -- +# silently, and reported as a cancelled run rather than a failure. The suites +# no longer need serialising anyway; each brings up its own attestation fixture +# under its own `FIXTURE_NS`, so there is nothing left to collide over. +# +# `cancel-in-progress` is the ordinary meaning here: a new push supersedes the +# run for the commit it replaced. +concurrency: + group: gateway-e2e-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + +jobs: + gateway-e2e: + runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} + # A cold musl build dominates; the suite itself waits out real ACME orders + # and a 20s cluster-sync settle, so it is minutes rather than seconds. + timeout-minutes: 45 + steps: + - uses: actions/checkout@v5 + + - name: Install Rust + uses: dtolnay/rust-toolchain@1.92.0 + with: + # run-e2e.sh builds a static gateway so the image can be alpine. + targets: x86_64-unknown-linux-musl + + - name: Install musl toolchain + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends musl-tools + + # The gateways create real WireGuard interfaces inside their containers, + # so the host kernel has to offer the module. Record it: a run that failed + # because the runner image changed under us should say so plainly instead + # of looking like a gateway bug. + - name: Record kernel capabilities + run: | + echo "kernel: $(uname -r)" + sudo modprobe wireguard 2>&1 || echo "no wireguard module available" + echo "wireguard loaded: $(lsmod | grep -c '^wireguard ' || true)" + docker version --format 'docker: {{.Server.Version}}' + docker compose version + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + dstack/target + key: gateway-e2e-${{ runner.os }}-${{ hashFiles('dstack/Cargo.lock') }} + restore-keys: gateway-e2e-${{ runner.os }}- + + # --keep-running so the containers survive for the log step below; the + # script's own EXIT trap would otherwise tear them down and leave nothing + # to collect from exactly the runs worth diagnosing. + - name: Gateway e2e suite + working-directory: dstack/gateway/test-run/e2e + run: ./run-e2e.sh --keep-running + + - name: Collect container logs on failure + if: failure() + working-directory: dstack/gateway/test-run/e2e + # The compose files require `${FIXTURE_NS:?}`, and the suite script + # exports it only to itself. Unset here, every `docker compose` call in + # this step -- not just the fixture one -- fails at interpolation, and + # the step still passes, so the artifact carries no container logs. + env: + FIXTURE_NS: dstack-fixture-e2e + run: | + # By compose service, not by container name: the shared attestation + # fixture deliberately does not pin one, so that more than one suite + # can be up at a time. + mkdir -p /tmp/gateway-e2e-logs + for svc in $(docker compose config --services); do + docker compose logs --no-color "$svc" > "/tmp/gateway-e2e-logs/$svc.log" 2>&1 || true + done + # The attestation fixture is a project of its own now, so it is not in + # this suite's service list and its logs have to be asked for + # separately -- they are where a quote-verification failure explains + # itself. + docker compose -p "$FIXTURE_NS" -f ../attestation/fixture.yml logs \ + --no-color > "/tmp/gateway-e2e-logs/fixture.log" 2>&1 || true + docker compose ps -a > /tmp/gateway-e2e-logs/compose-ps.txt 2>&1 || true + + - name: Tear down + if: always() + working-directory: dstack/gateway/test-run/e2e + run: ./run-e2e.sh down + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: gateway-e2e-logs + path: /tmp/gateway-e2e-logs/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/gateway-proxy-tests.yml b/.github/workflows/gateway-proxy-tests.yml new file mode 100644 index 000000000..86413a4c1 --- /dev/null +++ b/.github/workflows/gateway-proxy-tests.yml @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +name: Gateway proxy tests + +# The gateway's proxy data path has two opt-in optimisations (`tcp_splice`, +# `ktls`) whose behaviour depends on kernel capabilities and on a per-connection +# gate. Unit tests cover the relay functions; this runs a real gateway process +# and asserts on what actually reaches the wire. +# +# The suite runs in containers. One of them has a seccomp profile that makes +# `setsockopt(IPPROTO_TCP, TCP_ULP)` fail, which is how the no-TLS-ULP fallback +# is exercised now -- the suite used to `rmmod tls`, which needed passwordless +# sudo, took the module from the whole host, and skipped itself whenever +# anything else was using TLS. +on: + push: + branches: [ next, 'release/**' ] + paths: + - 'dstack/gateway/**' + - 'dstack/vendor/ktls/**' + # The suite runs the gateway with attestation on, so it depends on the + # shared fixture: the simulator, the mock collateral service, and the + # fixture payloads baked into the simulator image. + - 'dstack/guest-agent-simulator/**' + - 'dstack/crates/mock-attestation/**' + - 'sdk/simulator/**' + - '.github/workflows/gateway-proxy-tests.yml' + pull_request: + branches: [ next, 'release/**' ] + paths: + - 'dstack/gateway/**' + - 'dstack/vendor/ktls/**' + # The suite runs the gateway with attestation on, so it depends on the + # shared fixture: the simulator, the mock collateral service, and the + # fixture payloads baked into the simulator image. + - 'dstack/guest-agent-simulator/**' + - 'dstack/crates/mock-attestation/**' + - 'sdk/simulator/**' + - '.github/workflows/gateway-proxy-tests.yml' + +# Per workflow, not shared with the other two gateway suites. +# +# A group shared across all three does not serialise them, it drops one: a +# concurrency group holds a single *pending* run, so when the third workflow +# queued behind the first two, the one already waiting was cancelled -- +# silently, and reported as a cancelled run rather than a failure. The suites +# no longer need serialising anyway; each brings up its own attestation fixture +# under its own `FIXTURE_NS`, so there is nothing left to collide over. +# +# `cancel-in-progress` is the ordinary meaning here: a new push supersedes the +# run for the commit it replaced. +concurrency: + group: gateway-proxy-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + +jobs: + proxy-integration: + runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} + # Each of the ~25 arms restarts the gateway, and the idle-timeout arms wait + # out a real timeout. On top of that the suite now builds the shared + # attestation fixture, whose two images compile Rust from a cold docker cache + # on every run -- the cargo cache above does not reach inside a docker build. + timeout-minutes: 45 + steps: + - uses: actions/checkout@v5 + + - name: Install Rust + uses: dtolnay/rust-toolchain@1.92.0 + with: + # The container image is built around a static binary. + targets: x86_64-unknown-linux-musl + + - name: Install musl toolchain + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends musl-tools + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + dstack/target + key: gateway-proxy-${{ runner.os }}-${{ hashFiles('dstack/Cargo.lock') }} + restore-keys: gateway-proxy-${{ runner.os }}- + + - name: Record kernel capabilities + # The suite adapts to what the kernel offers, so the log needs to say + # what it had: a run that skipped kTLS looks the same as one that + # covered it otherwise. + run: | + echo "kernel: $(uname -r)" + # Without this the suite falls back to a dummy link and skips + # test_accel_status entirely -- including the only positive proof that + # splice engaged. The other two workflows already modprobe it. + sudo modprobe wireguard 2>&1 || echo "no wireguard module available" + sudo modprobe tls 2>&1 || echo "no TLS ULP available" + echo "tls module loaded: $(lsmod | grep -c '^tls ' || true)" + grep -B2 -A3 'gcm(aes)' /proc/crypto | grep -E '^(driver|priority)' \ + | paste - - | sort -u || true + + - name: Proxy integration tests + working-directory: dstack/gateway/test-run/proxy-e2e + run: ./run-proxy-tests.sh + + # No `docker compose logs` here: the suite is driven with `compose run + # --rm`, so its containers are deleted on exit and the script's own trap + # has already run `compose down -v`. Everything worth reading is in the + # bind-mounted work directory, which the suite made host-readable before + # exiting. + - name: Make suite logs readable + if: failure() + working-directory: dstack/gateway/test-run/proxy-e2e + run: | + mkdir -p run + docker run --rm -v "$PWD/run:/r" alpine:latest chmod -R a+rX /r || true + ls -R run || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: gateway-proxy-test-logs + # Only the logs. The work directory also holds each arm's throwaway + # TLS key, and an artifact is the wrong place for key material even + # when it is disposable. + path: dstack/gateway/test-run/proxy-e2e/run/*/logs/ + if-no-files-found: ignore + retention-days: 7 + + - name: Tear down + if: always() + working-directory: dstack/gateway/test-run/proxy-e2e + run: ./run-proxy-tests.sh down diff --git a/.github/workflows/gateway-release.yml b/.github/workflows/gateway-release.yml index 07d12796f..998209bc5 100644 --- a/.github/workflows/gateway-release.yml +++ b/.github/workflows/gateway-release.yml @@ -6,6 +6,11 @@ name: Gateway Release on: workflow_dispatch: + inputs: + version: + description: Existing release tag version without the gateway-v prefix + required: true + type: string push: tags: - 'gateway-v*' @@ -19,15 +24,30 @@ jobs: build-and-release: runs-on: ubuntu-latest steps: + - name: Parse and check version + env: + DISPATCH_VERSION: ${{ inputs.version }} + run: | + if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then + VERSION=$DISPATCH_VERSION + else + case "$GITHUB_REF" in + refs/tags/gateway-v*) VERSION=${GITHUB_REF#refs/tags/gateway-v} ;; + *) echo "Expected a gateway-v* tag" >&2; exit 1 ;; + esac + fi + if [[ ! "$VERSION" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "Version must be a valid Docker tag" >&2 + exit 1 + fi + echo "VERSION=$VERSION" >> "$GITHUB_ENV" + echo "RELEASE_TAG=gateway-v$VERSION" >> "$GITHUB_ENV" + + # Manual runs rebuild an existing release tag, not the selected branch. - name: Checkout repository uses: actions/checkout@v5 - - - name: Parse version from tag - run: | - # Extract version from tag (e.g., gateway-v1.2.3 -> 1.2.3) - VERSION=${GITHUB_REF#refs/tags/gateway-v} - echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "Parsed version: $VERSION" + with: + ref: ${{ github.event_name == 'workflow_dispatch' && env.RELEASE_TAG || github.sha }} - name: Log in to Docker Hub uses: docker/login-action@v3 @@ -38,27 +58,31 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Get Git commit timestamps - run: | - echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV - echo "GIT_REV=$(git rev-parse HEAD)" >> $GITHUB_ENV + - name: Resolve the revision to build + run: echo "GIT_REV=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + # Release images are built through the same script contributors run by + # hand, so a published image can be reproduced and checked digest-for-digest. + # See the builder README for the reproduction recipe. - name: Build and push Docker image id: build-and-push - uses: docker/build-push-action@v5 env: - SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }} - with: - context: dstack/gateway/dstack-app/builder - push: true - tags: ${{ vars.DOCKERHUB_ORG }}/dstack-gateway:${{ env.VERSION }} - platforms: linux/amd64 - provenance: false - build-contexts: | - build-shared=dstack/build/shared - build-args: | - DSTACK_REV=${{ env.GIT_REV }} - SOURCE_DATE_EPOCH=${{ env.TIMESTAMP }} + PUSH: "1" + METADATA_FILE: ${{ runner.temp }}/build-metadata.json + GIT_REV: ${{ env.GIT_REV }} + IMAGE_VERSION: ${{ env.VERSION }} + IMAGE_NAME: ${{ vars.DOCKERHUB_ORG }}/dstack-gateway:${{ env.VERSION }} + IMAGE_SOURCE_URL: ${{ github.server_url }}/${{ github.repository }} + DSTACK_SRC_URL: ${{ github.server_url }}/${{ github.repository }}.git + run: | + dstack/gateway/dstack-app/builder/build-image.sh \ + "$IMAGE_NAME" + digest=$(jq -r '."containerimage.digest"' "$METADATA_FILE") + if [ -z "$digest" ] || [ "$digest" = null ]; then + echo "the build did not report an image digest" >&2 + exit 1 + fi + echo "digest=$digest" >> "$GITHUB_OUTPUT" - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 @@ -70,6 +94,8 @@ jobs: - name: GitHub Release uses: softprops/action-gh-release@v2 with: + tag_name: ${{ env.RELEASE_TAG }} + target_commitish: ${{ env.GIT_REV }} name: "Gateway Release v${{ env.VERSION }}" body: | ## Docker Image Information diff --git a/.github/workflows/guest-os.yml b/.github/workflows/guest-os.yml index dc53eb24d..503159a7e 100644 --- a/.github/workflows/guest-os.yml +++ b/.github/workflows/guest-os.yml @@ -2,7 +2,9 @@ # # SPDX-License-Identifier: Apache-2.0 -name: Build Guest Images +# DEPRECATED: the Yocto backend is superseded by mkosi. Guest OS releases are +# built by mkosi-build.yml; this workflow is kept only to rebuild legacy images. +name: Build Guest Images (Yocto, deprecated) on: workflow_dispatch: @@ -23,8 +25,8 @@ jobs: - name: Checkout uses: actions/checkout@v5 - - name: Build production guest OS - run: make os-image + - name: Build production guest OS (deprecated Yocto backend) + run: make os-image-yocto - name: Upload artifacts uses: actions/upload-artifact@v4 diff --git a/.github/workflows/kms-release.yml b/.github/workflows/kms-release.yml index 0b33c5f88..122d886b6 100644 --- a/.github/workflows/kms-release.yml +++ b/.github/workflows/kms-release.yml @@ -6,6 +6,11 @@ name: KMS Release on: workflow_dispatch: + inputs: + version: + description: Existing release tag version without the kms-v prefix + required: true + type: string push: tags: - 'kms-v*' @@ -19,8 +24,30 @@ jobs: build-and-release: runs-on: ubuntu-latest steps: + - name: Parse and check version + env: + DISPATCH_VERSION: ${{ inputs.version }} + run: | + if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then + VERSION=$DISPATCH_VERSION + else + case "$GITHUB_REF" in + refs/tags/kms-v*) VERSION=${GITHUB_REF#refs/tags/kms-v} ;; + *) echo "Expected a kms-v* tag" >&2; exit 1 ;; + esac + fi + if [[ ! "$VERSION" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "Version must be a valid Docker tag" >&2 + exit 1 + fi + echo "VERSION=$VERSION" >> "$GITHUB_ENV" + echo "RELEASE_TAG=kms-v$VERSION" >> "$GITHUB_ENV" + + # Manual runs rebuild an existing release tag, not the selected branch. - name: Checkout repository uses: actions/checkout@v5 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && env.RELEASE_TAG || github.sha }} - name: Checkout contract dependencies run: | @@ -29,13 +56,6 @@ jobs: dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable \ dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades - - name: Parse version from tag - run: | - # Extract version from tag (e.g., kms-v1.2.3 -> 1.2.3) - VERSION=${GITHUB_REF#refs/tags/kms-v} - echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "Parsed version: $VERSION" - - name: Log in to Docker Hub uses: docker/login-action@v3 with: @@ -45,28 +65,31 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Get Git commit timestamps - run: | - echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV - echo "GIT_REV=$(git rev-parse HEAD)" >> $GITHUB_ENV + - name: Resolve the revision to build + run: echo "GIT_REV=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + # Release images are built through the same script contributors run by + # hand, so a published image can be reproduced and checked digest-for-digest. + # See the builder README for the reproduction recipe. - name: Build and push Docker image id: build-and-push - uses: docker/build-push-action@v5 env: - SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }} - with: - context: dstack/kms/dstack-app/builder - push: true - tags: ${{ vars.DOCKERHUB_ORG }}/dstack-kms:${{ env.VERSION }} - platforms: linux/amd64 - provenance: false - build-contexts: | - build-shared=dstack/build/shared - build-args: | - DSTACK_REV=${{ env.GIT_REV }} - DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }} - SOURCE_DATE_EPOCH=${{ env.TIMESTAMP }} + PUSH: "1" + METADATA_FILE: ${{ runner.temp }}/build-metadata.json + GIT_REV: ${{ env.GIT_REV }} + IMAGE_VERSION: ${{ env.VERSION }} + IMAGE_NAME: ${{ vars.DOCKERHUB_ORG }}/dstack-kms:${{ env.VERSION }} + IMAGE_SOURCE_URL: ${{ github.server_url }}/${{ github.repository }} + DSTACK_SRC_URL: ${{ github.server_url }}/${{ github.repository }}.git + run: | + dstack/kms/dstack-app/builder/build-image.sh \ + "$IMAGE_NAME" + digest=$(jq -r '."containerimage.digest"' "$METADATA_FILE") + if [ -z "$digest" ] || [ "$digest" = null ]; then + echo "the build did not report an image digest" >&2 + exit 1 + fi + echo "digest=$digest" >> "$GITHUB_OUTPUT" - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 @@ -87,6 +110,8 @@ jobs: - name: GitHub Release uses: softprops/action-gh-release@v2 with: + tag_name: ${{ env.RELEASE_TAG }} + target_commitish: ${{ env.GIT_REV }} name: "KMS Release v${{ env.VERSION }}" files: | dstack/kms/auth-eth/out/DstackKms.sol/DstackKms.json diff --git a/.github/workflows/mkosi-build.yml b/.github/workflows/mkosi-build.yml new file mode 100644 index 000000000..258cee143 --- /dev/null +++ b/.github/workflows/mkosi-build.yml @@ -0,0 +1,378 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +name: Build Guest Images (mkosi) + +# Publishing is bound to the tag, not to a workflow input: pushing +# mkosi-os-v is the only event that creates a GitHub Release. Every +# other trigger builds and reports the hashes without publishing anything, so a +# dry run can never be mistaken for a release. +on: + workflow_dispatch: + inputs: + repro_check: + description: 'Build twice with different job counts and compare byte-for-byte (roughly doubles the run time)' + required: false + default: false + type: boolean + pull_request: + paths: + - 'os/**' + - '.github/workflows/mkosi-build.yml' + # No paths filter here on purpose. GitHub ANDs the path filter with the ref + # filter, so a release tag placed on a commit that happens not to touch os/** + # would be silently dropped. The static job costs seconds and the image build + # is gated by its own `if`, so an unfiltered push trigger is cheap. + push: + branches: [next, 'release/**'] + tags: ['mkosi-os-v*'] + +concurrency: + group: mkosi-build-${{ github.ref }} + # A release build must never be cancelled by a later push. + cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/') }} + +jobs: + static: + name: Static contract + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@v5 + # Runs in seconds and covers the whole os/mkosi static contract, so it + # gates every change rather than waiting for a manual dispatch. + - name: Check the static contract + run: ./os/mkosi/build.sh lint + + build: + name: Build guest images + # The full image build takes ~40 minutes, so keep it off the PR path. + if: >- + github.event_name == 'workflow_dispatch' || + startsWith(github.ref, 'refs/tags/mkosi-os-v') + needs: static + runs-on: ubuntu-latest + timeout-minutes: 360 + permissions: + contents: read + outputs: + version: ${{ steps.hashes.outputs.version }} + os_image_hash: ${{ steps.hashes.outputs.os_image_hash }} + bare_sha256: ${{ steps.hashes.outputs.bare_sha256 }} + uki_sha256: ${{ steps.hashes.outputs.uki_sha256 }} + + steps: + # GitHub-hosted runners have enough CPU and memory for this build, but + # their preinstalled SDKs consume most of the available disk. Remove only + # those unused SDKs before checkout; the OS build itself remains hermetic. + - name: Reclaim runner disk space + run: | + sudo rm -rf \ + /opt/ghc \ + /opt/hostedtoolcache/CodeQL \ + /usr/local/.ghcup \ + /usr/local/lib/android \ + /usr/local/share/boost \ + /usr/share/dotnet + df -h / + + # Check out the dispatched ref, not a hardcoded revision. Pinning a + # revision here meant the job could never validate the branch it ran on, + # and the pinned commit becomes unreachable once the branch is squashed + # or rebased, which would break the job outright. + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Set up Python for mkosi + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Install pinned mkosi + run: | + set -euo pipefail + # Take the revision from versions.env so CI and the container builder + # in os/mkosi/repro-build cannot drift onto different mkosi builds. + # shellcheck source=/dev/null + source os/mkosi/versions.env + python3 -m venv "$RUNNER_TEMP/mkosi-venv" + "$RUNNER_TEMP/mkosi-venv/bin/pip" install --disable-pip-version-check \ + "mkosi @ git+https://github.com/systemd/mkosi.git@${MKOSI_REVISION}" + echo "$RUNNER_TEMP/mkosi-venv/bin" >> "$GITHUB_PATH" + + - name: Install mkosi host dependencies + run: | + mapfile -t dependencies < <(mkosi --directory os/mkosi dependencies) + sudo apt-get update + sudo apt-get install --yes --no-install-recommends "${dependencies[@]}" + + - name: Build guest images + id: build + env: + # ubuntu-latest currently provides four vCPUs and 16 GiB of RAM. + # Do not oversubscribe either the CPU or compiler memory. + JOBS: '4' + REPRO_CHECK: ${{ inputs.repro_check }} + run: | + set -euo pipefail + # GitHub-hosted Ubuntu runners restrict unprivileged user namespaces. + # Run mkosi as root instead of weakening the runner's AppArmor policy. + # Keep setup-python out of the build PATH because Ubuntu's lddtree + # expects the distro Python and its python3-pyelftools module. + ci_bin="$RUNNER_TEMP/mkosi-ci-bin" + mkdir -p "$ci_bin" + ln -s "$(command -v mkosi)" "$ci_bin/mkosi" + build_dir="$RUNNER_TEMP/mkosi-build" + if [ "$REPRO_CHECK" = "true" ]; then + # repro-check builds prod twice with different job counts and + # compares the release tarballs byte for byte. It leaves leg a in + # $build_dir/a, which is the same release contract as `image`. + action=repro-check + dist_dir="$build_dir/a" + else + action=image + dist_dir="$build_dir/out/prod" + fi + # A release build never reuses the component cache. The runner is + # fresh, so this costs nothing here, but it keeps the published + # artifacts a product of the cold path the repro check verifies. + sudo --set-home env \ + "PATH=$ci_bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + "JOBS=$JOBS" \ + ./os/mkosi/build.sh --no-cache "$action" "$build_dir" + sudo chown -R "$USER:$USER" "$build_dir" + echo "dist_dir=$dist_dir" >> "$GITHUB_OUTPUT" + df -h / + + - name: Build dstack-mr + # Only needed to decode the measurement CBOR for the report below. The + # image build produces its own copy under a scratch directory that + # mkosi.postoutput deletes, and this target directory stays outside the + # work tree so it cannot perturb a later source-tree build. + env: + CARGO_TARGET_DIR: ${{ runner.temp }}/dstack-mr-target + run: | + set -euo pipefail + # Take the toolchain from versions.env rather than pinning it again + # here, so this stays the same rustc the image build itself used. + # shellcheck source=/dev/null + source os/mkosi/versions.env + rustup toolchain install "$RUST_TOOLCHAIN_VERSION" --profile minimal + cargo "+$RUST_TOOLCHAIN_VERSION" build --release --locked \ + --manifest-path dstack/Cargo.toml -p dstack-mr + + - name: Report image hashes + id: hashes + env: + DIST_DIR: ${{ steps.build.outputs.dist_dir }} + DSTACK_MR: ${{ runner.temp }}/dstack-mr-target/release/dstack-mr + run: | + set -euo pipefail + # shellcheck source=/dev/null + source os/mkosi/versions.env + version="$DSTACK_VERSION" + bare="$DIST_DIR/dstack-$version.tar.gz" + uki="$DIST_DIR/dstack-$version-uki.tar.gz" + test -f "$bare" + test -f "$uki" + + # Read the identity out of the published tarball rather than the + # staging tree beside it, so the reported hashes describe exactly what + # ships. + work=$(mktemp -d) + tar -xzf "$bare" -C "$work" + img="$work/dstack-$version" + os_image_hash=$(cat "$img/digest.txt") + + # digest.txt is the release's claim; recompute it. os_image_hash is + # sha256(sha256sum.txt), the identity registered on chain and enforced + # by dstack-kms, so a mismatch here is a corrupt release. + computed=$(sha256sum "$img/sha256sum.txt" | awk '{print $1}') + if [ "$os_image_hash" != "$computed" ]; then + echo "::error::digest.txt says $os_image_hash but sha256sum.txt hashes to $computed" + exit 1 + fi + # Both packages describe one image and must carry one identity. + uki_digest=$(tar -xOf "$uki" "dstack-$version/digest.txt") + if [ "$uki_digest" != "$os_image_hash" ]; then + echo "::error::UKI package digest $uki_digest does not match bare-metal $os_image_hash" + exit 1 + fi + + bare_sha256=$(sha256sum "$bare" | awk '{print $1}') + uki_sha256=$(sha256sum "$uki" | awk '{print $1}') + git_revision=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["git_revision"])' "$img/metadata.json") + mrtd=$("$DSTACK_MR" inspect-measurement "$img/measurement.tdx.cbor" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["tdvf"]["mrtd"]["single_pass"])') + uki_auth=$("$DSTACK_MR" inspect-measurement "$img/measurement.gcp.cbor" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["uki_auth"])') + aws_pcr=$("$DSTACK_MR" inspect-measurement "$img/measurement.aws.cbor" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["boot_pcr_digest"])') + + { + echo "version=$version" + echo "os_image_hash=$os_image_hash" + echo "bare_sha256=$bare_sha256" + echo "uki_sha256=$uki_sha256" + } >> "$GITHUB_OUTPUT" + + # os_image_hash is the value operators need most, so surface it in the + # log, in the job summary, and as a downloadable record. + echo "::notice title=os_image_hash::$os_image_hash" + { + echo "### dstack guest OS \`$version\` (mkosi backend)" + echo + echo '| Field | Value |' + echo '| --- | --- |' + echo "| \`os_image_hash\` | \`$os_image_hash\` |" + echo "| Git revision | \`$git_revision\` |" + echo "| TDX \`MRTD\` | \`$mrtd\` |" + echo "| GCP \`uki_auth\` | \`$uki_auth\` |" + echo "| AWS \`boot_pcr_digest\` | \`$aws_pcr\` |" + echo "| \`dstack-$version.tar.gz\` | \`$bare_sha256\` |" + echo "| \`dstack-$version-uki.tar.gz\` | \`$uki_sha256\` |" + echo + echo '
sha256sum.txt (os_image_hash preimage)' + echo + echo '```' + cat "$img/sha256sum.txt" + echo '```' + echo + echo '
' + echo + echo '
metadata.json' + echo + echo '```json' + cat "$img/metadata.json" + echo '```' + echo + echo '
' + } >> "$GITHUB_STEP_SUMMARY" + + { + echo "os_image_hash $os_image_hash" + echo "git_revision $git_revision" + echo "tdx_mrtd $mrtd" + echo "gcp_uki_auth $uki_auth" + echo "aws_boot_pcr $aws_pcr" + echo "$bare_sha256 dstack-$version.tar.gz" + echo "$uki_sha256 dstack-$version-uki.tar.gz" + } | tee "$DIST_DIR/image-hashes.txt" + + - name: Upload guest images + uses: actions/upload-artifact@v4 + with: + name: mkosi-guest-images + path: | + ${{ steps.build.outputs.dist_dir }}/dstack-*.tar.gz + ${{ steps.build.outputs.dist_dir }}/image-hashes.txt + retention-days: 30 + if-no-files-found: error + + release: + name: Publish release + # The tag is the publish decision. Nothing else reaches this job. + if: startsWith(github.ref, 'refs/tags/mkosi-os-v') + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: release + permissions: + contents: write + + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Download guest images + uses: actions/download-artifact@v4 + with: + name: mkosi-guest-images + path: dist + + - name: Validate the release tag against the built image + env: + TAG: ${{ github.ref_name }} + OS_IMAGE_HASH: ${{ needs.build.outputs.os_image_hash }} + run: | + set -euo pipefail + echo "$TAG" | grep -Eq '^mkosi-os-v[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.-]+)?$' + version="${TAG#mkosi-os-v}" + # shellcheck source=/dev/null + source os/mkosi/versions.env + if [ "$version" != "$DSTACK_VERSION" ]; then + echo "::error::tag $TAG does not match DSTACK_VERSION=$DSTACK_VERSION in os/mkosi/versions.env" + exit 1 + fi + bare="dist/dstack-$version.tar.gz" + uki="dist/dstack-$version-uki.tar.gz" + test -f "$bare" + test -f "$uki" + + read -r image_version image_revision < <( + tar -xOf "$bare" "dstack-$version/metadata.json" | + python3 -c 'import json, sys; d=json.load(sys.stdin); print(d["version"], d["git_revision"])' + ) + test "$image_version" = "$version" + # A release must be reproducible from the tagged tree, so the revision + # baked into the measured metadata.json has to be the tagged commit. + # build.sh appends -modified when it builds a dirty work tree, which + # this equality also rejects. + if [ "$image_revision" != "$GITHUB_SHA" ]; then + echo "::error::image was built from $image_revision but the tag points at $GITHUB_SHA" + exit 1 + fi + # The UKI package is the GCP/AWS delivery path; assert its payload. + tar -tzf "$uki" | grep -Fx "dstack-$version/disk.raw" + test "$(tar -xOf "$bare" "dstack-$version/digest.txt")" = "$OS_IMAGE_HASH" + + - name: Create the release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.ref_name }} + VERSION: ${{ needs.build.outputs.version }} + OS_IMAGE_HASH: ${{ needs.build.outputs.os_image_hash }} + BARE_SHA256: ${{ needs.build.outputs.bare_sha256 }} + UKI_SHA256: ${{ needs.build.outputs.uki_sha256 }} + run: | + set -euo pipefail + { + echo "Guest OS image \`$VERSION\`, built by the Debian/mkosi backend." + echo + echo "Register \`os_image_hash\` on chain to authorize this image; see" + echo "[docs/onchain-governance.md](https://github.com/${GITHUB_REPOSITORY}/blob/${TAG}/docs/onchain-governance.md)." + echo + echo '| Field | Value |' + echo '| --- | --- |' + echo "| \`os_image_hash\` | \`$OS_IMAGE_HASH\` |" + echo "| Git revision | \`$GITHUB_SHA\` |" + echo "| \`dstack-$VERSION.tar.gz\` | \`$BARE_SHA256\` |" + echo "| \`dstack-$VERSION-uki.tar.gz\` | \`$UKI_SHA256\` |" + echo + echo "\`image-hashes.txt\` additionally records the TDX \`MRTD\`, the GCP" + echo "\`uki_auth\` hash and the AWS \`boot_pcr_digest\` for this image." + } > release-notes.md + # --verify-tag: the tag is the publish trigger, so it must already + # exist. This job never creates one. + # + # --prerelease is kept until release maintainers promote a stable + # mkosi-built release. os/mkosi is the default backend and Yocto is + # deprecated, but dropping the flag is a release decision, not a + # backend-status one. + # + # The title names the product, not the build backend: the tag prefix + # only exists to keep this workflow's trigger distinct from the + # deprecated Yocto one, and it should not leak into the release listing. + gh release create "$TAG" \ + "dist/dstack-$VERSION.tar.gz" \ + "dist/dstack-$VERSION-uki.tar.gz" \ + dist/image-hashes.txt \ + --title "dstack OS v$VERSION" \ + --notes-file release-notes.md \ + --verify-tag \ + --prerelease diff --git a/.github/workflows/prek-check.yml b/.github/workflows/prek-check.yml index 2524e6802..267f21c55 100644 --- a/.github/workflows/prek-check.yml +++ b/.github/workflows/prek-check.yml @@ -6,9 +6,9 @@ name: Prek checks on: push: - branches: [ master, next, dev-* ] + branches: [ next, 'release/**' ] pull_request: - branches: [ master, next, dev-* ] + branches: [ next, 'release/**' ] permissions: contents: read @@ -49,4 +49,16 @@ jobs: - name: Run prek checks (push) if: github.event_name == 'push' - run: prek run --from-ref ${{ github.event.before }} --to-ref ${{ github.event.after }} --show-diff-on-failure + env: + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.event.after }} + run: | + # A push that creates a branch reports an all-zero "before", and a + # force-push can name one this clone no longer has. Neither is a commit, + # so the range would be invalid; check every file instead. + if git rev-parse --verify --quiet "$BEFORE^{commit}" >/dev/null; then + prek run --from-ref "$BEFORE" --to-ref "$AFTER" --show-diff-on-failure + else + echo "no usable base commit ($BEFORE); checking all files" + prek run --all-files --show-diff-on-failure + fi diff --git a/.github/workflows/qemu-acpi-differential.yml b/.github/workflows/qemu-acpi-differential.yml new file mode 100644 index 000000000..de8dea222 --- /dev/null +++ b/.github/workflows/qemu-acpi-differential.yml @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +name: QEMU ACPI differential tests + +on: + pull_request: + branches: [next] + paths: + - .github/workflows/qemu-acpi-differential.yml + - dstack/crates/qemu-acpi/** + workflow_dispatch: + inputs: + seed: + description: Random seed accepted by Python int(value, 0) + required: false + type: string + cases: + description: Number of random cases in addition to boundary cases + required: false + default: "32" + type: string + +permissions: + contents: read + +jobs: + differential: + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + working-directory: dstack + env: + REFERENCE_IMAGE: kvin/dstack-acpi-tables@sha256:54e692d8c68c6f02dd7c655bf6de6e7f3ad7a43fded6ca00d8e998918108a3b4 + steps: + - uses: actions/checkout@v5 + + - name: Install Rust + uses: dtolnay/rust-toolchain@1.92.0 + + - name: Build Rust ACPI generator + run: cargo build -p qemu-acpi --example dump + + - name: Run boundary and seeded random differential tests + env: + INPUT_SEED: ${{ inputs.seed }} + INPUT_CASES: ${{ inputs.cases }} + run: | + seed="${INPUT_SEED:-0x$(git rev-parse --short=16 HEAD)}" + cases="${INPUT_CASES:-32}" + echo "ACPI differential seed: $seed" + echo "Random case count: $cases" + crates/qemu-acpi/scripts/differential-random.py \ + --image "$REFERENCE_IMAGE" \ + --rust-dump target/debug/examples/dump \ + --seed "$seed" \ + --cases "$cases" \ + --failure-dir qemu-acpi-failure + + - name: Upload failing blobs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: qemu-acpi-differential-failure + path: dstack/qemu-acpi-failure + if-no-files-found: ignore diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a269d4c49..0cb18deec 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -6,9 +6,9 @@ name: Rust checks on: push: - branches: [ master, next, dev-* ] + branches: [ next, 'release/**' ] pull_request: - branches: [ master, next, dev-* ] + branches: [ next, 'release/**' ] env: CARGO_TERM_COLOR: always @@ -21,17 +21,63 @@ jobs: working-directory: dstack steps: - uses: actions/checkout@v5 + with: + # The change detection below diffs against the base commit. + fetch-depth: 0 + + # Which files changed, decided here rather than with a workflow-level + # `paths:` filter. + # + # `rust-checks` is a required status check. A workflow skipped by `paths:` + # reports nothing at all, so the check never arrives and the pull request + # waits on it forever -- the filter has to live inside a job that always + # runs and always reports. Everything expensive below is gated on this + # output; when nothing relevant moved the job costs a checkout and a diff. + - name: Detect relevant changes + id: changes + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + if [ -z "$BASE_SHA" ]; then + echo "relevant=true" >> "$GITHUB_OUTPUT" + echo "push build: running everything" + exit 0 + fi + changed=$(git diff --name-only "$BASE_SHA...HEAD") + echo "$changed" + if echo "$changed" | grep -qE '^(dstack/|sdk/simulator/|rust-toolchain\.toml|\.github/workflows/rust\.yml)'; then + echo "relevant=true" >> "$GITHUB_OUTPUT" + else + echo "relevant=false" >> "$GITHUB_OUTPUT" + echo "nothing this job covers changed" + fi - name: Install Rust uses: dtolnay/rust-toolchain@1.92.0 with: components: clippy, rustfmt + # The largest job in the repository was compiling the workspace from nothing + # on every run. `simulator-release.yml` already caches this way; this brings + # the check that gates every merge in line with it. + # + # `workspaces` because the manifest lives in `dstack/`, not at the root, and + # the action keys the cache on the lockfile it finds there. `sdk/` is a + # separate workspace, cached by its own job. + - name: Cache cargo + if: steps.changes.outputs.relevant == 'true' + uses: Swatinem/rust-cache@v2 + with: + workspaces: dstack + - name: Run Clippy + if: steps.changes.outputs.relevant == 'true' run: cargo clippy -- -D warnings -D clippy::expect_used -D clippy::unwrap_used --allow unused_variables - name: Cargo fmt check + if: steps.changes.outputs.relevant == 'true' run: cargo fmt --check --all - name: Run tests + if: steps.changes.outputs.relevant == 'true' run: ./run-tests.sh diff --git a/.github/workflows/sdk-compat.yaml b/.github/workflows/sdk-compat.yaml new file mode 100644 index 000000000..2fdc127a6 --- /dev/null +++ b/.github/workflows/sdk-compat.yaml @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: © 2025 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +name: SDK compatibility + +permissions: + contents: read + +# Released SDKs against the current agent: nothing to re-check unless the SDKs +# or the agent they talk to changed. Not a required status check, so a +# workflow-level filter is safe. +# +# `dstack/**` rather than the guest-agent directories alone. The agent under +# test is built through `simulator_start`, whose dependency closure reaches a +# dozen crates in that workspace; naming the three obvious ones would leave the +# rest silently uncovered the moment one of them changed behaviour. +on: + push: + branches: [next, 'release/**'] + paths: + - 'sdk/**' + - 'dstack/**' + - 'rust-toolchain.toml' + - '.github/workflows/sdk-compat.yaml' + pull_request: + branches: [next, 'release/**'] + paths: + - 'sdk/**' + - 'dstack/**' + - 'rust-toolchain.toml' + - '.github/workflows/sdk-compat.yaml' + +env: + CARGO_TERM_COLOR: always + # Both the current tree and the released tags pin channel "1.92" in + # rust-toolchain.toml. Without this, rustup would treat that as a toolchain + # distinct from the "1.92.0" installed below and re-download it -- along with + # the three cross-compilation targets the pin asks for, none of which the SDK + # suites need. The compat job is about the wire surface, not about + # reproducing each tag's toolchain provisioning. + RUSTUP_TOOLCHAIN: 1.92.0 + +jobs: + # The agent the released SDKs are tested against, built once. + # + # Both matrix legs call `simulator_start`, which builds + # `dstack-guest-agent-simulator` out of the current tree -- the same binary, + # from the same commit, compiled twice in parallel. Nothing here replaces that + # call or hands the legs a binary from elsewhere: they still build from their + # own checkout, so what they test is still what this commit produces. This job + # only puts the artifacts in the cache first, under a key both legs restore, + # so the build they run finds its work already done. + # + # A `needs:` rather than a skip switch in `simulator_build` on purpose. That + # switch would be a path where the binary under test did not come from the + # checkout, which is not worth trading for a few minutes. + simulator: + name: Build the agent under test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install Rust + uses: dtolnay/rust-toolchain@1.92.0 + + - name: Cache cargo + uses: Swatinem/rust-cache@v2 + with: + workspaces: dstack + shared-key: sdk-compat-agent + + - name: Build the simulator + run: ./sdk/simulator/build.sh + + sdk-compat: + name: ${{ matrix.tag }} SDKs vs current agent + needs: simulator + runs-on: ubuntu-latest + strategy: + # Each tag is an independent claim; one failing should not hide the other. + fail-fast: false + matrix: + # v0.5.11 is the tag the freeze is defined against, and v0.5.8 is the + # newest tag whose SDK *clients and suites* actually differ from it. + # + # The bar is a different client, not a different tree. v0.5.10's sdk/ + # tree is the same object as v0.5.11's (`git rev-parse v0.5.10:sdk + # v0.5.11:sdk` prints one hash twice) and v0.5.9 differs from it by a + # single line -- a reqwest dependency spec in sdk/rust/Cargo.toml, with + # every client and test source in all four languages byte-identical. + # Either pair runs the same suites twice and reports one agreement as + # two independent results, at the cost of a second uncached + # four-language build. v0.5.8 is the first tag going back that carries + # a genuinely different client (26 files), and it earns its slot: it is + # the only released client that still asserts `EmitEvent` succeeds, so + # it is the only one that catches 0.6.0 removing it. v0.5.11's three + # assertions about that method were all rewritten in v0.5.9..v0.5.11 + # to tolerate failure under a simulator endpoint, so the v0.5.11 leg + # passes them without exercising anything. See sdk/compat/README.md. + tag: [v0.5.8, v0.5.11] + steps: + - uses: actions/checkout@v5 + with: + # The job checks the released SDKs out with `git worktree add `, + # which needs that tag's commit and its full tree locally. A shallow + # clone plus `fetch-tags` gives the refs but not a guarantee about the + # objects behind them, so take the whole history: the clone is a + # rounding error next to the Rust build in this job. + fetch-depth: 0 + + - name: Install Rust + uses: dtolnay/rust-toolchain@1.92.0 + + # Restore only. The `simulator` job above populated this key; a leg that + # saved as well would race its twin for the same entry and gain nothing, + # since the next run's `simulator` job writes it again anyway. + - name: Restore the agent build + uses: Swatinem/rust-cache@v2 + with: + workspaces: dstack + shared-key: sdk-compat-agent + save-if: false + + - name: Install Go + uses: actions/setup-go@v5 + with: + # The `go` directive in the released sdk/go/go.mod. + go-version: '1.24' + + - name: Install Node + uses: actions/setup-node@v5 + with: + # The released sdk/js/package.json asks for node >=18; 20 is what the + # JS SDK release workflow publishes from. + node-version: '20' + + - name: Install Python + uses: actions/setup-python@v5 + with: + # The released sdk/python/pyproject.toml asks for >=3.10; 3.11 is what + # the Python SDK release workflow builds with. The runner script + # installs PDM if it is missing, the same way sdk/run-tests.sh does. + python-version: '3.11' + + - name: Released SDKs against the current agent + run: ./sdk/compat/run-compat-tests.sh ${{ matrix.tag }} diff --git a/.github/workflows/sdk.yaml b/.github/workflows/sdk.yaml index 32a824e17..550806ff9 100644 --- a/.github/workflows/sdk.yaml +++ b/.github/workflows/sdk.yaml @@ -9,9 +9,9 @@ permissions: on: push: - branches: [master, next, dev-*] + branches: [next, 'release/**'] pull_request: - branches: [master, next, dev-*] + branches: [next, 'release/**'] env: CARGO_TERM_COLOR: always @@ -21,6 +21,44 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 + with: + # The change detection below diffs against the base commit. + fetch-depth: 0 + + # Which files changed, decided here rather than with a workflow-level + # `paths:` filter. + # + # `sdk-tests` is a required status check. A workflow skipped by `paths:` + # reports nothing at all, so the check never arrives and the pull request + # waits on it forever -- the filter has to live inside a job that always + # runs and always reports. Everything expensive below is gated on this + # output; when nothing relevant moved the job costs a checkout and a diff. + # + # `dstack/` is in the pattern because `sdk/run-tests.sh` starts the + # simulator, which is built from that workspace -- these suites exercise + # the agent's wire surface, not just the client libraries. The whole + # directory rather than the simulator's dependency closure: the closure is + # a dozen crates deep and would go stale the first time a dependency + # moved, and a filter that silently stops covering something is worse than + # one that occasionally runs when it need not. + - name: Detect relevant changes + id: changes + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + if [ -z "$BASE_SHA" ]; then + echo "relevant=true" >> "$GITHUB_OUTPUT" + echo "push build: running everything" + exit 0 + fi + changed=$(git diff --name-only "$BASE_SHA...HEAD") + echo "$changed" + if echo "$changed" | grep -qE '^(sdk/|dstack/|rust-toolchain\.toml|\.github/workflows/sdk\.yaml)'; then + echo "relevant=true" >> "$GITHUB_OUTPUT" + else + echo "relevant=false" >> "$GITHUB_OUTPUT" + echo "nothing this job covers changed" + fi - name: Install Rust uses: dtolnay/rust-toolchain@1.92.0 @@ -29,14 +67,34 @@ jobs: # This additional target is needed for wasm32 compatibility check. targets: wasm32-unknown-unknown, thumbv6m-none-eabi + # Both workspaces: `sdk/rust` for the SDK crates, and `dstack` for the + # release simulator that `sdk/run-tests.sh` builds before any suite runs. + # Without the second entry that build starts cold every time and is most + # of the job's wall time. + - name: Cache cargo + if: steps.changes.outputs.relevant == 'true' + uses: Swatinem/rust-cache@v2 + with: + workspaces: | + sdk/rust + dstack + # Save only from pushes to protected branches; pull requests restore + # from those. This cache is about 1 GB, and one per pull request ref + # would quickly push the repository past its cache quota and evict + # the entries that are actually shared. + save-if: ${{ github.event_name == 'push' }} + - name: SDK tests + if: steps.changes.outputs.relevant == 'true' run: cd sdk && ./run-tests.sh - name: Verify WASM compilation + if: steps.changes.outputs.relevant == 'true' # Ensures SDK types can be used in smart contracts run: cargo check --manifest-path sdk/rust/Cargo.toml --target=wasm32-unknown-unknown -p dstack-sdk-types - name: Verify no_std compatibility + if: steps.changes.outputs.relevant == 'true' run: | cargo test --manifest-path sdk/rust/Cargo.toml -p dstack-sdk-types --test no_std_test --no-default-features cargo check --manifest-path sdk/rust/Cargo.toml -p no_std_check --target thumbv6m-none-eabi diff --git a/.github/workflows/spdx-check.yml b/.github/workflows/spdx-check.yml index 1c9e6e990..dc24dfd08 100644 --- a/.github/workflows/spdx-check.yml +++ b/.github/workflows/spdx-check.yml @@ -6,9 +6,9 @@ name: SPDX License Check on: push: - branches: [ master, main ] + branches: [ next, 'release/**' ] pull_request: - branches: [ master, main ] + branches: [ next, 'release/**' ] jobs: reuse-lint: diff --git a/.github/workflows/verifier-release.yml b/.github/workflows/verifier-release.yml index d02d5b3c2..857a6382c 100644 --- a/.github/workflows/verifier-release.yml +++ b/.github/workflows/verifier-release.yml @@ -6,6 +6,11 @@ name: Verifier Release on: workflow_dispatch: + inputs: + version: + description: Existing release tag version without the verifier-v prefix + required: true + type: string push: tags: - 'verifier-v*' @@ -19,14 +24,30 @@ jobs: build-and-release: runs-on: ubuntu-latest steps: + - name: Parse and check version + env: + DISPATCH_VERSION: ${{ inputs.version }} + run: | + if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then + VERSION=$DISPATCH_VERSION + else + case "$GITHUB_REF" in + refs/tags/verifier-v*) VERSION=${GITHUB_REF#refs/tags/verifier-v} ;; + *) echo "Expected a verifier-v* tag" >&2; exit 1 ;; + esac + fi + if [[ ! "$VERSION" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "Version must be a valid Docker tag" >&2 + exit 1 + fi + echo "VERSION=$VERSION" >> "$GITHUB_ENV" + echo "RELEASE_TAG=verifier-v$VERSION" >> "$GITHUB_ENV" + + # Manual runs rebuild an existing release tag, not the selected branch. - name: Checkout repository uses: actions/checkout@v5 - - - name: Parse version from tag - run: | - VERSION=${GITHUB_REF#refs/tags/verifier-v} - echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "Parsed version: $VERSION" + with: + ref: ${{ github.event_name == 'workflow_dispatch' && env.RELEASE_TAG || github.sha }} - name: Log in to Docker Hub uses: docker/login-action@v3 @@ -37,31 +58,33 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Get Git commit timestamps - run: | - echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV - echo "GIT_REV=$(git rev-parse HEAD)" >> $GITHUB_ENV + - name: Resolve the revision to build + run: echo "GIT_REV=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + # Release images are built through the same script contributors run by + # hand, so a published image can be reproduced and checked digest-for-digest. + # See the builder README for the reproduction recipe. - name: Build and push Docker image id: build-and-push - uses: docker/build-push-action@v5 env: - SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }} - with: - context: dstack/verifier - file: dstack/verifier/builder/Dockerfile - push: true - tags: | - ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:${{ env.VERSION }} - ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:latest - platforms: linux/amd64 - provenance: false - build-contexts: | - build-shared=dstack/build/shared - build-args: | - DSTACK_REV=${{ env.GIT_REV }} - DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}.git - SOURCE_DATE_EPOCH=${{ env.TIMESTAMP }} + PUSH: "1" + METADATA_FILE: ${{ runner.temp }}/build-metadata.json + GIT_REV: ${{ env.GIT_REV }} + IMAGE_VERSION: ${{ env.VERSION }} + IMAGE_NAME: ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:${{ env.VERSION }} + LATEST_IMAGE_NAME: ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:latest + IMAGE_SOURCE_URL: ${{ github.server_url }}/${{ github.repository }} + DSTACK_SRC_URL: ${{ github.server_url }}/${{ github.repository }}.git + run: | + dstack/verifier/builder/build-image.sh \ + "$IMAGE_NAME" \ + "$LATEST_IMAGE_NAME" + digest=$(jq -r '."containerimage.digest"' "$METADATA_FILE") + if [ -z "$digest" ] || [ "$digest" = null ]; then + echo "the build did not report an image digest" >&2 + exit 1 + fi + echo "digest=$digest" >> "$GITHUB_OUTPUT" - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 @@ -73,6 +96,8 @@ jobs: - name: GitHub Release uses: softprops/action-gh-release@v2 with: + tag_name: ${{ env.RELEASE_TAG }} + target_commitish: ${{ env.GIT_REV }} name: "Verifier Release v${{ env.VERSION }}" body: | ## Docker Image Information diff --git a/.github/workflows/vmm-ui.yml b/.github/workflows/vmm-ui.yml index ea571b4ce..259865541 100644 --- a/.github/workflows/vmm-ui.yml +++ b/.github/workflows/vmm-ui.yml @@ -7,11 +7,19 @@ name: VMM UI build permissions: contents: read +# Only the VMM's web UI is built here. Not a required status check, so a +# workflow-level filter is safe. on: push: - branches: [ master, next, dev-* ] + branches: [ next, 'release/**' ] + paths: + - 'dstack/vmm/ui/**' + - '.github/workflows/vmm-ui.yml' pull_request: - branches: [ master, next, dev-* ] + branches: [ next, 'release/**' ] + paths: + - 'dstack/vmm/ui/**' + - '.github/workflows/vmm-ui.yml' jobs: build: diff --git a/CHANGELOG.md b/CHANGELOG.md index c88d03653..f94c67e96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,99 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added +- sdk: every SDK ships two clients, one per guest-agent surface, and nothing else. `DstackClientV0` speaks the frozen v0.5.11 API on the unversioned paths -- every method a v0.5.x SDK had, including `verify()` (which calls the server `Verify` RPC) and `emit_event()` (which surfaces the agent's removal error verbatim). `DstackClientV1` speaks `dstack.guest.v1` at `/v1` and has exactly its six methods: `issue_cert`, `get_key`, `attest`, `attest_gpu`, `info`, `version`. They are transport mirrors, not a compatibility layer: neither translates a call to the other, and each one's method set is its surface's method set. **v1 derives different key material than v0 for the same inputs** -- see `docs/guest-api-v1.md` for the migration. `TappdClient` is untouched. + + The blockchain adapters (Ethereum/viem, Solana) remain **v0-only** and stay typed against the v0 `GetKeyResponse`. v1 has no chain-related surface, deliberately: it returns key material, and what an application builds from those bytes is its own business rather than something the SDKs model. + + **The unsuffixed client name now means v1** in every SDK -- `DstackClient` is `DstackClientV1`, and it is the recommended default rather than a deprecated alias. Code that used the unsuffixed client for v0 calls fails loudly on upgrade, because the v1 method signatures differ and `get_key` requires `algorithm` explicitly; it does not silently derive different keys. To stay on the frozen surface, name `DstackClientV0`, which remains available and is marked legacy. + + Neither client ships a local signature-chain verifier. An SDK mirrors an API surface, and verifying needs no client and no connection -- `docs/guest-api-v1.md` specifies the rules normatively, down to the trust anchor, which is what a relying party implements against rather than four hand-ports of one byte format. `DstackClientV0.verify()` remains for single signatures, because that is what the frozen surface offers; v1 has no counterpart +- guest-agent: a versioned API, `dstack.guest.v1`, covering both trust surfaces. `DstackGuest` is served at `/v1` on the internal socket (`IssueCert`, `GetKey`, `Attest`, `AttestGpu`, `Info`, `Version`); `Worker` at `/prpc/v1` on the external listener (`Info`, `Version`, `Health`). They are two services rather than one mounted twice because the two listeners have different reachability: the internal socket answers only the app itself and hands out key material, the external one answers anyone who can route to the CVM and never does. The scheme is uniform across both listeners: `/v0` and `/v1` on the internal socket, `/prpc/v0` and `/prpc/v1` on the external one, where `v0` is the frozen v0.5.11 surface. The historical unversioned paths (`/` and `/prpc`) stay mounted as aliases onto the same frozen handlers, so a pre-0.6 client keeps working unchanged and cannot drift from `/v0` -- they are additional mounts, not a parallel implementation. Version selection is by URL path alone -- no header negotiation, no default-version redirect -- so a request URL is the whole record of which contract the caller asked for. `Tappd` predates this scheme and is untouched. Specified byte-for-byte in `docs/guest-api-v1.md`, which is the normative reference an implementation is written from, and the constants it specifies -- salt, both context tags, the length-prefixed encoding, the claim -- live in `ra-tls`'s `api_v1` module so the agent, the verifier and the coming SDK support link against one definition instead of three transcriptions of the prose +- guest-agent: v1 derives application keys under a real domain-separated KDF. v0 fed the path alone into HKDF and handed the same 32 bytes to both secp256k1 and ed25519, so the two curves shared one secret and the algorithm a caller asked for changed nothing about the key it got. v1 derives under its own HKDF salt (`dstack-guest-v1`, against the legacy `RATLS`) and binds a versioned context tag, the algorithm and the caller's `domain` as length-prefixed fields, so the curves never collide and no two names encode alike -- a path is arbitrary caller-chosen bytes, so any delimiter it could also contain is a collision waiting to happen. **v1 keys are therefore not v0 keys for the same name**, deliberately and with no compatibility mode; an app holding assets under a v0 key migrates them with a transaction signed by the old key. The separate salt matters because the legacy HKDF `info` is the caller's `path` verbatim: under a shared salt, a caller passing the v1 `info` byte string as a v0 `path` reproduced a v1 key exactly. That was never a privilege boundary -- same app, same root key, either surface reachable -- but a KDF whose separation depends on nobody choosing an awkward input is one refactor from separating nothing, and it costs nothing to close on a surface with no deployed keys. The derivation is flat: `a/b` is not a child of `a`, and there is no BIP-32-style hierarchy. Committed test vectors pin the output bytes +- guest-agent: v1's signature-chain claim cannot be forged through the v0 surface. v0's first link signs `keccak256("{purpose}:{hex(pubkey)}")` over a caller-chosen `purpose`, which lets a malicious app steer the app root key into signing nearly any ASCII string ending in `:` plus hex. The v1 claim is length-prefixed and binds the raw public key bytes, so it always contains `00` bytes inside the region a v0 preimage requires to be hex-only. The exclusion is structural rather than probabilistic, and a regression test builds the strongest available forgery and asserts it fails +- guest-agent: v1 ships no `Sign` and no `Verify`. The agent is not an HSM: anything that can reach this socket can ask `GetKey` for the private key, so a server-side `Sign` grants no capability its caller lacks and buys an IPC round trip and another entry point to audit. Verifying needs neither key nor attestation, and the agent's answer arrives unattested. Apps sign locally with a standard library; relying parties verify locally against the normative rules in `docs/guest-api-v1.md`, which specify the KDF, the claim encoding, and every verification step down to the trust anchor. Both RPCs stay on the unversioned surface for 0.5.x clients +- guest-agent: v1 `GetTlsKey` is renamed `IssueCert`, because certificate issuance is the operation -- the agent builds a CSR and relays it to the KMS `SignCert` flow. The returned private key is incidental: freshly generated per call, fed by none of the request fields, and unrelated to the app identity. Only the integrated one-step mode ships; a caller-supplied-CSR mode would arrive as new fields +- guest-agent: v1 `Info` returns identity and configuration only. The `tcb_info` JSON blob is gone: its measurement values are typed top-level fields that each appear exactly once, and the documents it nested (`app_compose`, `vm_config`) are served directly instead of through two layers of JSON parsing. MRTD, RTMR0-3 and the event log are deliberately absent -- they are attestation data, and `Info` handing out unattested copies invited relying parties to trust values nothing vouched for. Ask `Attest` and verify. The demo `app_cert` is dropped as well +- guest-agent: `AttestGpu` collects vendor-native GPU evidence on demand against a caller-supplied 32-byte nonce. It returns opaque, versioned evidence bundles identified by vendor and format for independent appraisal, and the response format is extensible to additional GPU vendors. v1 only +- guest-agent: v1 `Attest` accepts `include_boottime_gpu_evidence` and returns the boot-time GPU attestation evidence in `AttestResponse.boottime_gpu_evidence`, so a verifier fetches the attestation and the GPU evidence in one round trip. It arrives as a `GpuEvidenceBundle` list -- the same shape `AttestGpu` returns, so a consumer writes one bundle parser and dispatches on `format`: `nvidia-nvattest-boottime-json-v1` is the record written at boot, `nvidia-nvattest-collect-evidence-json-v1` is collected on demand against a caller's nonce, and a verifier for one does not appraise the other. Absence is the empty list. The bundle's `evidence` is the nvattest output byte for byte as read from disk, because the only thing binding it to the boot is sha256 over precisely those bytes against the measured `gpu-attestation` event. `Attest` is also v1's sole CVM attestation entry point: the `VersionedAttestation` it returns already carries the TDX quote and event log, and unlike `GetQuote` it answers on every supported platform +- guest-agent: v1 `Attest` and `IssueCert` always use the MessagePack V1 attestation schema. `Attest` returns it, and `IssueCert` puts it in the CSR, which the signer embeds in the RA-TLS certificate in the same form. Until now the encoding followed the attestation's contents: a TDX CVM whose runtime events were all V1 got the legacy SCALE form, and one with V2 events or a pod payload got MessagePack. That left every v1 client implementing two decoders for a surface that has no released client to stay compatible with. The frozen v0 `Attest` and `GetTlsKey` keep the content-dependent choice: relying parties before 0.5.9 decode only SCALE, and KMS onboarding embeds v0 `Attest` bytes in the certificate an older source KMS verifies. **Requires 0.5.9 or later** of the KMS signing a v1 `IssueCert` CSR and of anything verifying a v1 `Attest` result or a v1-issued RA-TLS certificate +- sdk: `AppCompose` in the Go SDK gained `init_script`, `storage_fs`, `swap_size`, `event_log_version`, `port_policy` and `verity_volumes`, and `Requirements` gained `gpu_policy` in the Go and Python SDKs +- shared API authentication (`dstack-api-auth`) protecting the full VMM HTTP/pRPC/UI surface and unifying Gateway/KMS admin auth: bearer/`X-Admin-Token`/HTTP Basic/bcrypt htpasswd, constant-time verification (#796) +- certbot/gateway: opt-in `dns-persist-01` certificate validation ([draft-ietf-acme-dns-persist-01](https://datatracker.ietf.org/doc/html/draft-ietf-acme-dns-persist-01)), which issues without a DNS provider credential at all. `dns-01` needs write access to the zone on every order, so a gateway CVM holds a Cloudflare API token for the life of the deployment -- a token that rewrites the whole zone, not just `_acme-challenge`. Under `dns-persist-01` the zone owner publishes one `_validation-persist.` TXT record naming the CA and the ACME account; nothing about it changes between orders, so certbot only ever reads DNS and the zone can be hosted anywhere, with no provider integration. Set `challenge = "dns-persist-01"` in `certbot.toml`, or `challenge` on a gateway ZT domain; the default stays `dns-01` and existing deployments are untouched. `certbot dns-records` prints the records to publish, `GetZtDomain`/`ListZtDomains` return them in `required_dns_records`, and the gateway logs them wherever it would otherwise have written DNS. CAA records carry `validationmethods=dns-persist-01` to match, so switching methods means republishing both. Two gateway operations change shape for such a domain: `SetCaa` skips it, having nothing to reconcile without write access, and `RotateAcmeCredentials` leaves it broken until the operator republishes -- the record still names the old account -- so the response now returns the new records in `required_dns_records`. A rotation that registers the account but fails to re-pin some domain's CAA now reports those domains in `repin_failed_domains` instead of failing the call: the account exists and the cluster is using it, so raising an error there reads as "nothing happened" and invites a retry that registers another account. **Experimental**: the draft is still changing, and Let's Encrypt serves the challenge on staging only pending an open working-group issue. Documented in `docs/certbot-dns-persist-01.md`. + + Two settings changed shape for every deployment, not only `dns-persist-01` ones. `issuer_domain_name` now names the CA in the CAA records written for `dns-01` too -- its default is `letsencrypt.org`, so an untouched configuration writes exactly what it wrote before -- and it is validated where it is set, since it lands verbatim in an RFC 8659 `issue-value` and CAA is republished by deleting the old records first. The pre-order DNS self-check is now capped at half of `renew_timeout` rather than at its own configured value: both defaulted to 300s in the gateway, and the CLI wrapped a 300s wait in a 120s budget, so the timeout around the order always fired first and a missing record was reported as `certificate request timed out` instead of by name +- gateway: `Admin.Status` reports `health_gating`, so an operator can see whether this node's health polling is switched on. With it off, instances that opted in sit at `unknown` forever and are all in rotation, which is otherwise indistinguishable on the dashboard from being held out pending a first answer +- gateway: `Admin.SetInstanceReady` takes a CVM instance out of its app's load-balancing rotation without stopping it; instance-id routing stays open so the instance can still be investigated, and the setting survives re-registration +- gateway: operator-set per-instance overrides now live under their own KV keys — `admin//ready` and `admin//port_policy` — instead of inside the instance record, so a CVM re-registration can no longer drop them and setting one cannot discard a peer's unsynced change to the other. An override left in an instance record by an earlier build is moved across on load +- gateway: opt-in application-level health polling. An app sets `requirements.health_check` in its app-compose; the gateway then asks that CVM's guest agent (new `Health` RPC on the v1 external surface, polled at `/prpc/v1/Health`) whether the app is serving, and keeps instances that say no -- or that have not answered since registering -- out of app-id load balancing. Apps that do not opt in are never polled. Instance-id routing is never gated, and an app whose every instance reports unhealthy is routed to anyway rather than blackholed. Verdicts are per-node and not persisted: a gateway restart puts the whole app back at `unknown` at once, which is exactly the case the fail-open covers. Documented in `docs/app-health-checks.md`. One unreleased skew: an interim `next` build that served `Health` at `/prpc` and registered with `health_check = true` will 404 every poll and drop out of app-id rotation after `failure_threshold` failures. Instance-id routing is unaffected and restarting on a current build clears it; no released agent is involved, since `Health` never shipped in a release +- app-compose: `requirements.health_status_file` names a file the app writes its own verdict into -- two lines, `healthy`/`unhealthy` and the unix timestamp it was written at, treated as unhealthy once older than 60s. It must be a regular file (a FIFO would park a thread of the agent's blocking pool on every refresh); symlinks are followed, and its contents are never quoted back into a report. Without it the agent judges the app's own Compose project: every container that declares a `healthcheck` must be running and healthy, and a project where *no* container declares one reports unhealthy rather than passing silently +- guest-agent: container health also covers the `nerdctl-compose` runner, read through `nerdctl inspect` (its output is Docker-compatible). Requires nerdctl >= 2.3.1 for Compose `healthcheck:` to be honoured; the mkosi backend is pinned to 2.3.5 +- http-client: a caller can bound the response body (`http_request_bounded`, `PrpcClient::with_max_response_bytes`). Nothing is bounded by default — `dstack vmm logs --lines 100000` is a legitimate multi-megabyte fetch — but every client that talks to a guest agent opts in, in the gateway and in the VMM, because a CVM is untrusted and one of them polls on a timer against the whole fleet + +### Fixed +- data disks: discard now propagates through ZFS or ext4, dm-crypt, virtio-blk, and QEMU so encrypted qcow2 images release deleted blocks instead of growing with lifetime writes. Discard defaults on and can be disabled with `storage_discard: false` when allocation-pattern leakage is unacceptable; upgrading an existing ZFS pool also starts a one-time trim for historical free space +- certbot: a certificate covering both a name and its wildcard (`example.com` and `*.example.com`) could never be issued over dns-01. The two authorizations are answered under one `_acme-challenge.example.com`, each with its own TXT value, and the publish step cleared every TXT record at that name before writing its own -- so the second authorization deleted the record answering the first, and the order failed with `Correct value not found for DNS challenge`. Clearing leftovers from an aborted run is now done once per challenge name per issuance, and the records for one name accumulate instead of replacing each other; cleanup afterwards is unchanged, deleting each record this run created by id +- certbot: editing `domains` in `certbot.toml` had no effect once a certificate existed. Issuance was skipped whenever `live/cert.pem` was present, whatever names it carried, and renewal read its name list back off that certificate rather than the configuration -- so an added or removed name never reached the CA, and the mismatch survived every renewal. The live certificate's DNS names are now compared against the configured list (as sets, case- and trailing-dot-insensitive) and a mismatch reissues, logging both lists. A reissue that fails does not take the renewal check down with it: a name the CA will not validate is reported on every cycle, while the certificate actually being served keeps renewing, and the failure is still what the run returns unless the renewal committed something of its own +- certbot: `certbot cfg` attached every comment after `cf_api_url` to the wrong key, because an absent optional field shifts the generated document's keys out of step with the struct's. The template's documentation is now looked up by key name +- gateway: a fresh cluster could register several ACME accounts at once. The shared account was registered lazily, on whichever renewal first found the credentials record empty, under no lock but the per-domain renewal lock -- so two domains, or two nodes, starting together each spent a rate-limited registration, and the last-writer-wins credentials record kept exactly one of them. The attestation written beside it races under its own key, so the account the cluster ends up using need not be the one it can prove it holds. One lock in the KV store now covers every operation over the shared account -- rotation, CAA reconciliation, and first-use registration -- so they are ordered across nodes and not merely within one process, as CAA reconciliation was before. Registration re-reads the record under that lock and adopts an account another node registered while it waited, and builds the DNS provider client only after the lock is granted, so a refused attempt costs no provider API call +- gateway: a node removed via `RemoveNode` silently rejoined the cluster the next time it started, because every node re-registers its own sync address on boot. Once tombstone GC is collecting, that comeback is worse than an annoyance: a stale data directory diverges from every digest, and the divergence repair's full re-exchange resurrects records whose deletes the cluster already collected. Removal now writes a durable marker — a live record, so the GC can never eat it — that every gateway's sync endpoints enforce; a removed node's envelopes are refused until an operator re-admits it with `SetNodeUrl`. The refused node counts HTTP 403 sync rejections (`dstack_gateway_sync_rejected_total`), including removal lockouts and app-identity mismatches. Every gateway also exposes `dstack_gateway_node_last_seen_timestamp_seconds` per known node, so a long-offline gateway is a one-line alert instead of an ack-watermark puzzle +- gateway: deleted KV records left a tombstone that nothing ever collected, so every deregistered CVM stayed on disk for the life of the deployment. Tombstones every peer has acknowledged are now dropped once every `tombstone_gc_writes` replicated writes (default 10000, zero disables); the trigger counts replicated writes rather than reading a clock, so nodes in a cluster collect in the same window without depending on time synchronization. A `SetTombstoneGcConfig` admin RPC stores an operator override in the KV itself, replicating one pace to every node +- gateway: `Admin.RemoveCvm` now reports the outcome of the `inst/` tombstone alone. A failure to delete associated override or telemetry records is logged instead of failing the call, so a removal that did take effect is no longer reported as failed — which also aborted the local routing cleanup that follows it. Re-issuing a removal still sweeps up override and telemetry records orphaned by an earlier partial failure +- sdk: `get_compose_hash` in the Python SDK mutated the dictionary it was given, stripping `docker_config` and `requirements` from it, so hashing the same manifest twice returned two different digests -- the second one for a manifest missing those blocks +- sdk: the Go compose-hash helper HTML-escaped `<`, `>` and `&`, so an app-compose carrying an `os_version` bound such as `">=0.6.1"` hashed differently in Go than in Rust, Python and JavaScript. Any digest Go produced for such a manifest was wrong, and that digest is what gets whitelisted on chain +- sdk: the Go SDK's `gpu_policy` silently dropped sub-fields it did not declare on a decode/re-hash round trip; it now carries them through like the rest of `Requirements` +- bound VMM-to-guest RPCs and guest-agent dependency calls to prevent stalled peers from retaining request resources indefinitely +- sdk: the Go and Python compose-hash helpers silently dropped every app-compose field they did not declare, so `getComposeHash` returned a digest for an app-compose that was not the one being deployed — and that digest is what gets whitelisted on chain. The missing fields are named above; both now keep unrecognised keys as well, so a guest that gains a field before the SDK does still hashes correctly +- sdk: the JavaScript and Rust clients report a non-2xx answer with its status *and* the server's message. The JS transport ignored the status entirely, and its unix-socket branch never even parsed the status line, so an agent with no `/v1` mount — every pre-0.6 agent — answered a v1 call with an HTML 404 page that reached the caller as `failed to parse response`, which names neither the status nor the cause. The Rust clients discarded the other half: `error_for_status()` keeps the status line and throws away the body the agent puts its reason in, and on the unix path the crate's JSON helper insists on deserializing the *error* body too, so the same HTML page came back as a JSON parse failure. Both now raise `HTTP : ` — the prpc `error` field when the body is one, otherwise the body itself, bounded so an error page cannot become the whole message. Python and Go already reported both halves, in their own wording; this aligns JS with Rust rather than all four with each other. + + **This changes two things beyond the error text.** In JS, six v0 methods that never checked the response body — `getKey`, `getTlsKey`, `info`, `version`, `sign`, and `TappdClient.deriveKey` — used to *resolve* on a prpc failure, handing back an object whose every field was `undefined` next to an `error` string. They now reject. `version()`, whose own documentation says it throws against an agent too old to have the RPC, previously resolved there; it now does what it says. In Rust, neither transport sends a duplicated `Content-Type` header any more: on the unix path the crate's JSON helper appended its own on top of the one the client passed, and on the HTTP path `reqwest`'s `json()` did the same, since `header()` appends rather than replaces. Rocket reads the first value, so the agent never noticed, but a stricter intermediary is entitled to refuse a request carrying the field twice. + +- sdk: fix two encoding defects in the JavaScript transport, both of which corrupted or stalled real calls. The unix-socket branch declared `Content-Length` as `payload.length` — UTF-16 code units — while writing UTF-8, so a request carrying any non-ASCII field sent more bytes than it declared: the agent parsed truncated JSON and the surplus poisoned the connection. `getKey`'s `domain` is specified to accept any byte string a proto3 `string` can carry, so this was reachable by design rather than by accident. On the read path the same branch compared that byte-counted `Content-Length` against a JS string's `.length`, a condition a multi-byte body can never satisfy, so the client waited out the agent's ten-second keep-alive instead of returning — one accented character in an app-compose comment turned `info()` into a ten-second call and made `isReachable()` report a healthy agent as unreachable. That branch no longer speaks HTTP by hand: it uses node's client over `socketPath`, which frames the request, de-chunks the response, and counts bytes where bytes are meant. Responses on both branches are now assembled as bytes and decoded once, so a UTF-8 sequence split across two TCP reads no longer decodes to replacement characters on each side — which the old code did while resolving successfully, handing back quietly wrong data. A timed-out call also says so: the timeout aborted the request before rejecting, and `abort()` runs its listener synchronously, so the abort's `request aborted` always won the race and `request timed out` was unreachable — leaving `isReachable()` unable to tell a hung agent from any other failure + +### Changed +- kms: client certificates are authenticated by the attestation they carry rather than by their issuer. Rocket configures mutual TLS through rustls' `WebPkiClientVerifier`, which pins a CA — but an RA-TLS certificate is self-issued and carries its identity in a TEE quote, so there is nothing to chain to. `GetTempCaCert` bridged the gap by handing every caller a shared CA private key purely so the minted certificate would chain somewhere; the CA established nothing (its key is public by design, and the endpoint is unauthenticated) and the check that has always carried the meaning is the quote verification that runs afterwards. The KMS now hands rustls a verifier that requires an attestation and ignores the issuer. Nothing changes for callers: guests and KMS-to-KMS onboarding still mint their client certificates from the temp CA, and those are now accepted for the attestation they carry. What changes is that the TLS layer went from admitting any certificate signed by a public key to requiring an attested one, and that a self-issued certificate is now accepted — which is what lets callers be migrated off `GetTempCaCert` in a follow-up. `[rpc.tls.mutual]` is no longer the trust anchor and is dropped from `kms.toml` and the KMS config templates; leaving it in an existing deployment's config is inert. The gateway's `[tls.mutual]` is unaffected — it pins the KMS root CA, which is a real trust anchor +- guest-agent: the `/metrics` exposition gains a conventions-compliant `dstack_guest_*` series set (application prefix; `_bytes`/`_seconds` unit suffixes; no `_total` suffix on gauges, which reads as a counter to every tool; the four OS/kernel/CPU gauges folded into one `dstack_guest_info`; `disk_used_ratio` 0–1 instead of a percentage). The old `system_*`/`disk_*` names are still emitted verbatim, marked deprecated, and will be removed in a future release — this endpoint is tenant-facing, so existing dashboards get a migration window +- dstack-util: a CVM re-registers with the gateway node that last accepted it, before falling back to the configured order. The list used to be walked from the top every time, so every CVM piled onto the first URL and the whole fleet snapped back to it the moment it recovered from an outage — and each move rewrites the instance record from a different node's memory +- vmm: optionally randomize the KMS and gateway URL orders written to each CVM's system configuration so new CVMs distribute their initial requests across service nodes; both are enabled by default in `vmm.toml` +- http-client: HTTP clients are built once and shared instead of per request, which is what every `http_request*` call did until now -- each one paid for a connection pool, a DNS resolver and a TLS configuration it then threw away. Callers choose whether requests may reuse a connection (`RequestOptions::connection_reuse`, `PrpcClient::with_connection_reuse`); the default is to reuse. The gateway's health poller opts out: opening the connection is half of what a probe asks, since an agent that has run out of file descriptors keeps serving connections it already has while refusing every new one -- and every connection the gateway proxies to an app is a new one +- os/yocto: nerdctl 2.2.1 → 2.3.5, so `nerdctl compose` honours the Compose `healthcheck:` field (only translated into `--health-*` flags from 2.3.1 on). Requires openembedded-core to move to `wrynose` head for go 1.26.5, which also brings gcc 15.2 → 15.3 — every guest image measurement changes, so the new image hashes need whitelisting in KMS +- guest-agent: both unversioned surfaces are closed at exactly v0.5.11, and every new capability goes to `dstack.guest.v1` instead. Everything added to them after v0.5.11 never shipped in a release, so it is removed rather than frozen in: `AttestGpu` and `Attest`'s GPU-evidence field on the internal service, `AttestAppKey` and `Health` on the external one. All of those are v1 features now. No released client is affected -- both services are now byte-identical to v0.5.11 apart from doc comments and `reserved` statements holding the interim field numbers, so an unreleased `next` build in a dev environment cannot have one of them silently absorbed by a future field. "Frozen except for additions" is how all of them arrived in the first place +- guest-agent: `GetQuote` is restricted to Intel TDX. It used to answer on every platform, returning an empty `quote` plus a `GetQuoteResponse.attestation` field carrying the versioned attestation — a shape only that one RPC produced, and one `Attest` already covers. Platforms without TDX now get an error telling them to call `Attest`, and the `attestation` field is gone from the RPC and from the Rust, Python, Go and JS SDKs. GCP Confidential VMs still get an answer — the gate is whether the platform has a TDX quote — but only the TDX half of one: `GetQuoteResponse` has no field for the vTPM quote GCP's verification also binds, so relying parties there want `Attest`, and the docs say so. `Tappd.TdxQuote`/`RawQuote` share the same backend path, so they fail closed there too instead of returning an empty quote +- guest-agent: `Worker.GetAttestationForAppKey` is **retained**, unchanged and frozen, and v1 ships no counterpart. The method attests the key v0's KDF derives at path `vms` with purpose `signing`, and no v1 `GetKey(domain, algorithm)` can return that key -- different salt, different `info`, no `purpose` input -- so a v1 counterpart would have handed a pure-v1 app an attestation of a public key whose private half it could not obtain, which is worse than having no method because it looks like it works. A v1 app attests its own key instead: derive it at `/v1/GetKey`, commit the public key into `report_data`, call `/v1/Attest`, and serve the result to relying parties itself. That is strictly more capable, since the app chooses which key and which commitment format rather than being limited to the one the agent would derive. Legacy flows keep using the frozen method; it remains Intel TDX only, because it returns a `GetQuoteResponse` +- sdk: the Go SDK's v1 `IssueCert` defaults `usage_server_auth` to true, as the Rust, Python and JavaScript v1 clients already did. Go was the odd one out, so the same argument-free call produced a certificate that could serve TLS in three languages and one that could not in the fourth — and a certificate you cannot serve with is useless to most callers. `WithCertUsageServerAuth(false)` opts out. v0's `GetTlsKey` keeps its `false` default deliberately: that is what the released 0.5.x Go SDK sent, and `DstackClientV0` mirrors released behaviour rather than the better choice +- sdk: the JavaScript v1 `issueCert` response no longer carries a raw-bytes accessor. `asUint8Array()` is **removed rather than renamed**: it existed to feed the private key into the blockchain adapters, and v1 has no chain-flavoured surface. `IssueCert` returns TLS material, PEM is the form a TLS stack takes, and a caller who genuinely needs DER converts it with a standard library. The Rust, Python and Go v1 clients already returned the PEM string and the chain alone, so all four now agree. v0's `GetTlsKeyResponse.asUint8Array` is untouched — released API, and the viem and solana adapters depend on its truncating behaviour +- sdk: the JavaScript v1 GPU evidence bundle's `asUint8Array()` accessor is gone, and `evidence` is the vendor's bytes directly, as Go's `Evidence` always was. Byte-exact off the wire, because sha256 over precisely those bytes is what the measured `gpu-attestation` event commits to +- sdk: every field the `dstack.guest.v1` proto declares `bytes` is now that language's byte type on the v1 clients — Rust `Vec`, Python `bytes`, JavaScript `Uint8Array`, Go `[]byte` — and the `decode_*` helpers are gone along with the hex strings they decoded. Eleven fields move: `GetKeyResponse`'s `key`, `public_key` and `signature_chain`; `AttestResponse.attestation`; `GpuEvidenceBundle.evidence`; and `InfoResponse`'s `app_id`, `instance_id`, `compose_hash`, `device_id`, `os_image_hash` and `mr_aggregated`. Rust's `AttestConfig.report_data` moves with them, so the public builder and `attest()` finally agree on a type. **The JSON wire is unchanged** — it still carries lowercase hex; the encoding moved into the serialization layer, as serde's `hex::serde` in Rust, an annotated pydantic type in Python, and the client's decode step in JavaScript. Go already did this and is untouched. + + The old typing did quiet damage. `docs/guest-api-v1.md` says of the v1 key claim that `public_key` is the raw derived public key, *not* a hex string, and its verification steps rebuild the claim from raw bytes — so a Rust or Python caller passing `response.public_key` straight into a claim builder built it over 66 ASCII characters instead of 33 bytes. No type error, no exception, just a chain that never verifies. `evidence` had the same shape of problem: three separate documents had to keep repeating "hash the decoded bytes, not the string as returned", precisely because the type did not say it. In Go the mistake was unspellable, and now it is unspellable everywhere -- which is why Rust's `report_data` is a `ReportData` newtype rather than a bare `Vec`: `&str` and `String` both implement `Into>`, so under the builder's `into` coercion `.report_data("00ff")` would still compile and attest the four ASCII bytes of that string. The newtype converts from a `Vec`, an array or a slice and from nothing else, so the ergonomics survive and the string does not. A `compile_fail` doctest keeps it that way. Nor was there one line to learn: before this, three of the eleven fields had no decoder at all in Rust, six had none in Python, and JavaScript had decoded three of them for a while. + + Decoding got stricter where it was silently lenient. JavaScript relied on `Buffer.from(value, 'hex')`, which stops at the first pair it cannot parse and returns the prefix, so a corrupted `app_id` became a short `Uint8Array` and a signature chain with one bad link came back quietly one link short; it now throws and names the field, as Rust, Python and Go already did. A required field that is absent altogether is an error rather than empty bytes -- `os_image_hash` and `mr_aggregated` are the two exceptions, read as empty so a degraded `Info` stays parseable, which is what Rust's `#[serde(default)]` already did and what Python now does instead of rejecting the response. Python also stops accepting hex with embedded whitespace, which `bytes.fromhex` skips and Rust refuses. + + The `borsh` encoding of these structs does change, since borsh writes a `Vec` as length-prefixed bytes where it wrote a hex `String` before. The v1 types shipped in no 0.5.x release, so the window is between 0.6 prereleases: a blob written by an earlier one deserializes without error into these types and yields the ASCII of the hex string. Only the JSON wire is compatible. + + The request direction follows: v1's `attest` and `attest_gpu` take bytes and nothing else in all four SDKs. Rust and Go always did; Python and JavaScript also accepted a string and UTF-8 encoded it, so `attest("deadbeef")` committed to the eight ASCII characters rather than the four bytes they spell, and `attestGpu` on a 32-character string passed the length check on its way to attesting the wrong nonce. Both now raise, and say whether to `encode()` the text or decode the hex. **Breaking for a v1 caller passing a string** -- but v1 has not shipped, and the v0 clients keep the old signature, so a 0.5.x program is unaffected. + + Decoding a malformed response now reaches the same verdict in all four SDKs. Differential testing -- 194 identical JSON bodies through four real clients -- found them agreeing on 143 and diverging on 51, every divergence in absence, `null`, or JSON type confusion rather than bad hex. Go read an absent or null `bytes` field as empty and returned a nil error, so an error body arriving with a 200 handed back a zero-length private key that looked like an answer; JavaScript's hex check stringified its argument, so `app_id: ["00112233"]` decoded to one attacker-chosen byte. Required fields are now required, an absent `os_image_hash` or `mr_aggregated` is still empty, an explicit `null` is malformed everywhere, and a bundle without the `vendor` a caller dispatches on is an error rather than evidence routed to no verifier. All 194 bodies now agree. None of them is reachable from a conforming agent, which emits every field, always lowercase hex, never `null` -- they are reachable from a compromised or non-dstack server, which is the threat model these fields already take seriously. + + **v0 deliberately keeps its hex strings and `decode_*` helpers.** That surface mirrors the released 0.5.x SDK so a 0.5.x program keeps working by changing only the class name; retyping every byte field would break that promise on an API that is frozen anyway. The blockchain adapters are v0-typed and unaffected +- sdk: the v0 modules carry a `_v0` suffix, so the file a reader opens matches the client it holds. Rust's `dstack_sdk::dstack_client` becomes `dstack_sdk::dstack_client_v0` and `dstack_sdk_types::dstack` becomes `dstack_sdk_types::dstack_v0`; Python's `dstack_sdk.dstack_client` becomes `dstack_sdk.dstack_client_v0`; Go's `client.go`/`client_test.go` become `client_v0.go`/`client_v0_test.go`; and the JavaScript `index.ts`, which held both surfaces in one file, splits into `client-v0.ts`, `client-v1.ts` and a `shared.ts`, leaving `index.ts` as a barrel that re-exports exactly the names it always did. Until now the unsuffixed *file* meant v0 while the unsuffixed *class* meant v1, so a reader opening `dstack_client.rs` for the recommended client found the legacy one instead. **There are deliberately no backward-compat module aliases**: 0.6.0 is the loud-break release, and an import of an old module path fails at build time rather than silently binding the frozen surface under a name that now means something else. Package-level exports are untouched in every SDK — `dstack_sdk::DstackClient`, `from dstack_sdk import DstackClientV0` and `@phala/dstack-sdk`'s public surface are exactly what they were; only a deep import of the module path moves. In Go this is file naming alone, since it is all one `package dstack` +- sdk: the v0 clients are deprecated in the way each language's tooling understands, not only in prose. Rust's `DstackClientV0` and `TappdClient` carry `#[deprecated(since = "0.6.0")]`, so a downstream build warns at every mention of the type — the `use`, the constructor, any signature naming it. Method calls on an already-built client stay silent, because Rust does not propagate the attribute to inherent methods. Python's `DstackClientV0` and `AsyncDstackClientV0` emit a `DeprecationWarning` on construction, through the same helper `TappdClient` already used, alongside the `.. deprecated:: 0.6.0` docstring note they already carried. JavaScript's `DstackClientV0` already had its `@deprecated` JSDoc and `TappdClient` gains one. Go's `// Deprecated:` markers were in place but seven sat mid-comment rather than as their own trailing paragraph, which is the only form gopls and pkg.go.dev recognise, and are repaired. + + Nothing is removed and the wire behaviour is unchanged, but Python's marker is a runtime warning rather than a build-time one: a downstream test suite that turns `DeprecationWarning` into an error (`filterwarnings = error`, which is a common setting) will fail on `DstackClientV0()` until it adds a filter. The frozen surface stays reachable under its explicit name; it just says what it is now + + +### Deprecated +- os: the Yocto guest-OS backend (`os/yocto/`) is deprecated in favor of mkosi (`os/mkosi/`), which is now the default and recommended backend. `os/build.sh` defaults to `--backend mkosi`, and `make os-image` / `make os-repro-check` build with mkosi. The Yocto builds move to `make os-image-yocto` / `make os-repro-check-yocto`; `make os-image-mkosi` / `make os-repro-check-mkosi` remain as aliases. Every Yocto entrypoint prints a deprecation warning, and the backend is kept only to rebuild existing Yocto images. The mkosi build still reads patches, units and scripts from `os/yocto/`, so the directory stays until those files move + +### Removed +- sdk: `TlsKeyOptions.path` in the JavaScript SDK. `GetTlsKeyArgs` has no such field and `getTlsKey` never read it, so a caller who set it was silently ignored. Breaking at the type level only, and only for code whose value was already being discarded. `deriveKey`'s `path` is a real, deprecated Tappd-era parameter and stays; the Python, Rust and Go v0 TLS-key options never carried one +- guest-agent: the `EmitEvent` RPC no longer records anything -- runtime RTMR3 events are system-owned in 0.6.0, so an app can no longer extend the measurement chain. The method itself stays on the unversioned path and always fails with an error naming the removal, rather than being deleted outright: a deleted method answers HTTP 404 `Service not found: EmitEvent`, which tells a 0.5.x caller nothing about why its events stopped being recorded, while the kept stub fails with a message naming the removal and pointing at `report_data`. **Breaking:** any app extending RTMR3 at runtime must stop; bind app data through `report_data` instead, which is what most callers wanted anyway +- gateway: `core.debug.insecure_skip_attestation`. It turned off both checks that make a gateway cluster a trust boundary: the node stopped asking its guest agent for its own app id, and every WaveKV sync and push, plus `ensure_from_gateway`, stopped checking the peer's. With it set, anything that could reach the sync routes could insert entries that replicated to every gateway in the cluster. It existed only because the integration suites could not run without it; they now run against a guest agent simulator and verify quotes through the production path, so nothing sets it. `docs/security/security-model.md` argues dstack's development switches are acceptable because they are visible in attestation measurements or public contract state -- that argument cannot cover the switch deciding whether attestation happens at all, which is why this one is deleted rather than documented. There is no replacement. **Breaking:** the config struct does not use `deny_unknown_fields`, so a leftover line is ignored rather than rejected, and what follows depends on why it was set. On a TDX host with a guest agent the gateway starts normally and peers that cannot present a verifiable app id stop being accepted. On a host with no guest agent -- the usual reason to have set it -- the gateway no longer starts at all, failing with `Failed to get app info`, because `my_app_id` is what every peer check compares against and a node that cannot learn its own identity must not come up without one. Remove the line and make sure every node can attest + + ## [0.5.5] - 2025-10-20 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index fb39cc38c..6fdcf5174 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -146,11 +146,12 @@ forge clean ### Logging and Error Messages -- **Never capitalize** the first letter of log messages and error messages +- Start log messages and error messages with lowercase text +- Preserve the conventional capitalization of identifiers and acronyms when they + begin a message (for example, `RIM`, `OCSP`, or `HTTP`) - Example: `log::info!("starting server on port {}", port);` - Example: `anyhow::bail!("failed to connect to server");` - -This rule is enforced in `.cursorrules`. +- Example: `log::warn!("RIM upstream is unavailable");` ## Key Security Concepts @@ -176,7 +177,11 @@ This rule is enforced in `.cursorrules`. ### Local Development Setup -1. Build guest-OS artifacts through `os/build.sh` (see `os/README.md`) +1. Build guest-OS artifacts with `make os-image`, which uses the default mkosi + backend (see `docs/building-guest-os.md`). The Yocto backend (`os/yocto/`, + `make os-image-yocto`, `--backend yocto`) is deprecated: never use it for new + images or features. Do not delete `os/yocto/` either: mkosi still reads + patches, units, and scripts from it (`grep -rn yocto os/mkosi`) 2. Download or build guest OS image 3. Run components in separate terminals: - KMS: `./dstack-kms -c kms.toml` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5b69bce63..c6cf272d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,29 @@ Thank you for your interest in contributing to this project! +## Branches + +- `next` is the integration mainline and the default branch. Open every pull + request against it unless a maintainer asks otherwise. +- `release/v..x` carries a released line — `release/v0.5.x` is + the current one. It only takes fixes cherry-picked back from `next`; do not + develop on it directly. Patch tags are cut here. + +Name a working branch whatever describes it. CI runs on pull requests, so a +branch gets its checks once a PR is open rather than on every push. + +The default branch was renamed from `master` to `next`. Web links, raw file +URLs, and the REST API redirect, but the old ref name is gone at the git +level: `git fetch origin master` and `git clone -b master` now fail. Update an +existing clone with: + +```bash +git branch -m master next +git fetch origin +git branch -u origin/next next +git remote set-head origin -a +``` + ## Development 1. Fork the repository @@ -62,6 +85,12 @@ dstack deploy ./docker-compose.yml \ --no-tee ``` +To exercise persistent TPM-backed app keys as well, install `swtpm` on the VMM +host and select `key_provider=tpm` in the VMM console (or pass `--key-provider +tpm` in tooling that exposes the compose option). The VMM keeps the software +TPM state in the VM work directory so that seal/unseal survives guest restarts. +The software TPM is host-controlled and does not provide hardware isolation. + The simulator package is installed only in development images. This mode provides no hardware isolation and must never be used with production workloads or secrets. Real quote generation, hardware isolation, and KMS diff --git a/LICENSES/LGPL-2.1-or-later.txt b/LICENSES/LGPL-2.1-or-later.txt new file mode 100644 index 000000000..c6487f4fd --- /dev/null +++ b/LICENSES/LGPL-2.1-or-later.txt @@ -0,0 +1,176 @@ +GNU LESSER GENERAL PUBLIC LICENSE + +Version 2.1, February 1999 + +Copyright (C) 1991, 1999 Free Software Foundation, Inc. +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts as the successor of the GNU Library Public License, version 2, hence the version number 2.1.] + +Preamble + +The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public Licenses are intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. + +This license, the Lesser General Public License, applies to some specially designated software packages--typically libraries--of the Free Software Foundation and other authors who decide to use it. You can use it too, but we suggest you first think carefully about whether this license or the ordinary General Public License is the better strategy to use in any particular case, based on the explanations below. + +When we speak of free software, we are referring to freedom of use, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish); that you receive source code or can get it if you want it; that you can change the software and use pieces of it in new free programs; and that you are informed that you can do these things. + +To protect your rights, we need to make restrictions that forbid distributors to deny you these rights or to ask you to surrender these rights. These restrictions translate to certain responsibilities for you if you distribute copies of the library or if you modify it. + +For example, if you distribute copies of the library, whether gratis or for a fee, you must give the recipients all the rights that we gave you. You must make sure that they, too, receive or can get the source code. If you link other code with the library, you must provide complete object files to the recipients, so that they can relink them with the library after making changes to the library and recompiling it. And you must show them these terms so they know their rights. + +We protect your rights with a two-step method: (1) we copyright the library, and (2) we offer you this license, which gives you legal permission to copy, distribute and/or modify the library. + +To protect each distributor, we want to make it very clear that there is no warranty for the free library. Also, if the library is modified by someone else and passed on, the recipients should know that what they have is not the original version, so that the original author's reputation will not be affected by problems that might be introduced by others. + +Finally, software patents pose a constant threat to the existence of any free program. We wish to make sure that a company cannot effectively restrict the users of a free program by obtaining a restrictive license from a patent holder. Therefore, we insist that any patent license obtained for a version of the library must be consistent with the full freedom of use specified in this license. + +Most GNU software, including some libraries, is covered by the ordinary GNU General Public License. This license, the GNU Lesser General Public License, applies to certain designated libraries, and is quite different from the ordinary General Public License. We use this license for certain libraries in order to permit linking those libraries into non-free programs. + +When a program is linked with a library, whether statically or using a shared library, the combination of the two is legally speaking a combined work, a derivative of the original library. The ordinary General Public License therefore permits such linking only if the entire combination fits its criteria of freedom. The Lesser General Public License permits more lax criteria for linking other code with the library. + +We call this license the "Lesser" General Public License because it does Less to protect the user's freedom than the ordinary General Public License. It also provides other free software developers Less of an advantage over competing non-free programs. These disadvantages are the reason we use the ordinary General Public License for many libraries. However, the Lesser license provides advantages in certain special circumstances. + +For example, on rare occasions, there may be a special need to encourage the widest possible use of a certain library, so that it becomes a de-facto standard. To achieve this, non-free programs must be allowed to use the library. A more frequent case is that a free library does the same job as widely used non-free libraries. In this case, there is little to gain by limiting the free library to free software only, so we use the Lesser General Public License. + +In other cases, permission to use a particular library in non-free programs enables a greater number of people to use a large body of free software. For example, permission to use the GNU C Library in non-free programs enables many more people to use the whole GNU operating system, as well as its variant, the GNU/Linux operating system. + +Although the Lesser General Public License is Less protective of the users' freedom, it does ensure that the user of a program that is linked with the Library has the freedom and the wherewithal to run that program using a modified version of the Library. + +The precise terms and conditions for copying, distribution and modification follow. Pay close attention to the difference between a "work based on the library" and a "work that uses the library". The former contains code derived from the library, whereas the latter must be combined with the library in order to run. + +GNU LESSER GENERAL PUBLIC LICENSE +TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + +0. This License Agreement applies to any software library or other program which contains a notice placed by the copyright holder or other authorized party saying it may be distributed under the terms of this Lesser General Public License (also called "this License"). Each licensee is addressed as "you". + +A "library" means a collection of software functions and/or data prepared so as to be conveniently linked with application programs (which use some of those functions and data) to form executables. + +The "Library", below, refers to any such software library or work which has been distributed under these terms. A "work based on the Library" means either the Library or any derivative work under copyright law: that is to say, a work containing the Library or a portion of it, either verbatim or with modifications and/or translated straightforwardly into another language. (Hereinafter, translation is included without limitation in the term "modification".) + +"Source code" for a work means the preferred form of the work for making modifications to it. For a library, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the library. + +Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running a program using the Library is not restricted, and output from such a program is covered only if its contents constitute a work based on the Library (independent of the use of the Library in a tool for writing it). Whether that is true depends on what the Library does and what the program that uses the Library does. + +1. You may copy and distribute verbatim copies of the Library's complete source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and distribute a copy of this License along with the Library. + +You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. + +2. You may modify your copy or copies of the Library or any portion of it, thus forming a work based on the Library, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a table of data to be supplied by an application program that uses the facility, other than as an argument passed when the facility is invoked, then you must make a good faith effort to ensure that, in the event an application does not supply such function or table, the facility still operates, and performs whatever part of its purpose remains meaningful. + +(For example, a function in a library to compute square roots has a purpose that is entirely well-defined independent of the application. Therefore, Subsection 2d requires that any application-supplied function or table used by this function must be optional: if the application does not supply it, the square root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Library, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Library, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library with the Library (or with a work based on the Library) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. + +3. You may opt to apply the terms of the ordinary GNU General Public License instead of this License to a given copy of the Library. To do this, you must alter all the notices that refer to this License, so that they refer to the ordinary GNU General Public License, version 2, instead of to this License. (If a newer version than version 2 of the ordinary GNU General Public License has appeared, then you can specify that version instead if you wish.) Do not make any other change in these notices. + +Once this change is made in a given copy, it is irreversible for that copy, so the ordinary GNU General Public License applies to all subsequent copies and derivative works made from that copy. + +This option is useful when you wish to copy part of the code of the Library into a program that is not a library. + +4. You may copy and distribute the Library (or a portion or derivative of it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange. + +If distribution of object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place satisfies the requirement to distribute the source code, even though third parties are not compelled to copy the source along with the object code. + +5. A program that contains no derivative of any portion of the Library, but is designed to work with the Library by being compiled or linked with it, is called a "work that uses the Library". Such a work, in isolation, is not a derivative work of the Library, and therefore falls outside the scope of this License. + +However, linking a "work that uses the Library" with the Library creates an executable that is a derivative of the Library (because it contains portions of the Library), rather than a "work that uses the library". The executable is therefore covered by this License. Section 6 states terms for distribution of such executables. + +When a "work that uses the Library" uses material from a header file that is part of the Library, the object code for the work may be a derivative work of the Library even though the source code is not. Whether this is true is especially significant if the work can be linked without the Library, or if the work is itself a library. The threshold for this to be true is not precisely defined by law. + +If such an object file uses only numerical parameters, data structure layouts and accessors, and small macros and small inline functions (ten lines or less in length), then the use of the object file is unrestricted, regardless of whether it is legally a derivative work. (Executables containing this object code plus portions of the Library will still fall under Section 6.) + +Otherwise, if the work is a derivative of the Library, you may distribute the object code for the work under the terms of Section 6. Any executables containing that work also fall under Section 6, whether or not they are linked directly with the Library itself. + +6. As an exception to the Sections above, you may also combine or link a "work that uses the Library" with the Library to produce a work containing portions of the Library, and distribute that work under terms of your choice, provided that the terms permit modification of the work for the customer's own use and reverse engineering for debugging such modifications. + +You must give prominent notice with each copy of the work that the Library is used in it and that the Library and its use are covered by this License. You must supply a copy of this License. If the work during execution displays copyright notices, you must include the copyright notice for the Library among them, as well as a reference directing the user to the copy of this License. Also, you must do one of these things: + + a) Accompany the work with the complete corresponding machine-readable source code for the Library including whatever changes were used in the work (which must be distributed under Sections 1 and 2 above); and, if the work is an executable linked with the Library, with the complete machine-readable "work that uses the Library", as object code and/or source code, so that the user can modify the Library and then relink to produce a modified executable containing the modified Library. (It is understood that the user who changes the contents of definitions files in the Library will not necessarily be able to recompile the application to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (1) uses at run time a copy of the library already present on the user's computer system, rather than copying library functions into the executable, and (2) will operate properly with a modified version of the library, if the user installs one, as long as the modified version is interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at least three years, to give the same user the materials specified in Subsection 6a, above, for a charge no more than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy from a designated place, offer equivalent access to copy the above specified materials from the same place. + + e) Verify that the user has already received a copy of these materials or that you have already sent this user a copy. + +For an executable, the required form of the "work that uses the Library" must include any data and utility programs needed for reproducing the executable from it. However, as a special exception, the materials to be distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. + +It may happen that this requirement contradicts the license restrictions of other proprietary libraries that do not normally accompany the operating system. Such a contradiction means you cannot use both them and the Library together in an executable that you distribute. + +7. You may place library facilities that are a work based on the Library side-by-side in a single library together with other library facilities not covered by this License, and distribute such a combined library, provided that the separate distribution of the work based on the Library and of the other library facilities is otherwise permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities. This must be distributed under the terms of the Sections above. + + b) Give prominent notice with the combined library of the fact that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. + +8. You may not copy, modify, sublicense, link with, or distribute the Library except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense, link with, or distribute the Library is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. + +9. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Library or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Library (or any work based on the Library), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Library or works based on it. + +10. Each time you redistribute the Library (or any work based on the Library), the recipient automatically receives a license from the original licensor to copy, distribute, link with or modify the Library subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties with this License. + +11. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Library at all. For example, if a patent license would not permit royalty-free redistribution of the Library by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply, and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. + +This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. + +12. If the distribution and/or use of the Library is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Library under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. + +13. The Free Software Foundation may publish revised and/or new versions of the Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Library does not specify a license version number, you may choose any version ever published by the Free Software Foundation. + +14. If you wish to incorporate parts of the Library into other free programs whose distribution conditions are incompatible with these, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. + +NO WARRANTY + +15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +END OF TERMS AND CONDITIONS + +How to Apply These Terms to Your New Libraries + +If you develop a new library, and you want it to be of the greatest possible use to the public, we recommend making it free software that everyone can redistribute and change. You can do so by permitting redistribution under these terms (or, alternatively, under the terms of the ordinary General Public License). + +To apply these terms, attach the following notices to the library. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. + + one line to give the library's name and an idea of what it does. + Copyright (C) year name of author + + This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the library, if necessary. Here is a sample; alter the names: + +Yoyodyne, Inc., hereby disclaims all copyright interest in +the library `Frob' (a library for tweaking knobs) written +by James Random Hacker. + +signature of Ty Coon, 1 April 1990 +Ty Coon, President of Vice +That's all there is to it! diff --git a/Makefile b/Makefile index f51b05896..ab7900f6f 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 +# Submodules for the deprecated Yocto backend. OS_YOCTO_SUBMODULES := \ os/yocto/deps/bitbake \ os/yocto/deps/openembedded-core \ @@ -12,7 +13,8 @@ OS_YOCTO_SUBMODULES := \ os/yocto/deps/meta-rust-bin \ os/yocto/deps/meta-security -.PHONY: help core core-check core-test sdk-test os os-yocto os-deps os-image os-repro-check +.PHONY: help core core-check core-test sdk-test os os-image os-repro-check \ + os-image-mkosi os-repro-check-mkosi os-yocto os-deps os-image-yocto os-repro-check-yocto help: @echo "dstack monorepo targets:" @@ -20,11 +22,15 @@ help: @echo " core-check check the Rust workspace" @echo " core-test test the Rust workspace with the simulator" @echo " sdk-test run all public SDK tests" - @echo " os build the guest OS natively with the default backend" - @echo " os-yocto build the guest OS natively with Yocto" - @echo " os-deps initialize only the Yocto dependency submodules" - @echo " os-image build one production guest image in the pinned container" - @echo " os-repro-check build twice and compare reproducible outputs" + @echo " os build the guest OS natively with the default (mkosi) backend" + @echo " os-image build one production guest image with mkosi in the pinned container" + @echo " os-repro-check build twice with mkosi and compare outputs byte for byte" + @echo "" + @echo "Deprecated Yocto targets (do not use for new work; use the mkosi targets above):" + @echo " os-yocto build the guest OS natively with Yocto" + @echo " os-deps initialize only the Yocto dependency submodules" + @echo " os-image-yocto build one production guest image with Yocto" + @echo " os-repro-check-yocto build twice with Yocto and compare outputs" core: cargo build --manifest-path dstack/Cargo.toml @@ -41,14 +47,27 @@ sdk-test: os: ./os/build.sh +# The mkosi backend vendors no submodules, so these do not depend on os-deps. +os-image: + ./os/mkosi/repro-build/repro-build.sh + +os-repro-check: + ./os/mkosi/repro-build/repro-build.sh -c + +# Aliases kept for existing callers. +os-image-mkosi: os-image + +os-repro-check-mkosi: os-repro-check + +# Deprecated Yocto backend. Kept only to rebuild existing Yocto images. os-yocto: ./os/build.sh --backend yocto os-deps: git submodule update --init --depth 1 -- $(OS_YOCTO_SUBMODULES) -os-image: os-deps +os-image-yocto: os-deps cd os/yocto/repro-build && ./repro-build.sh -n -os-repro-check: os-deps +os-repro-check-yocto: os-deps cd os/yocto/repro-build && ./repro-build.sh diff --git a/README.md b/README.md index cb3bdb6fb..e1e8ba940 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ### The open framework for confidential AI. [![GitHub Stars](https://img.shields.io/github/stars/dstack-tee/dstack?style=flat-square&logo=github)](https://github.com/Dstack-TEE/dstack/stargazers) -[![License](https://img.shields.io/github/license/dstack-tee/dstack?style=flat-square)](https://github.com/Dstack-TEE/dstack/blob/master/LICENSE) +[![License](https://img.shields.io/github/license/dstack-tee/dstack?style=flat-square)](https://github.com/Dstack-TEE/dstack/blob/next/LICENSE) [![REUSE status](https://api.reuse.software/badge/github.com/Dstack-TEE/dstack)](https://api.reuse.software/info/github.com/Dstack-TEE/dstack) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/Dstack-TEE/dstack) [![Telegram](https://img.shields.io/badge/Telegram-2CA5E0?style=flat-square&logo=telegram&logoColor=white)](https://t.me/+UO4bS4jflr45YmUx) @@ -73,6 +73,9 @@ Setting up dstack on your own hardware? Start with the [self-hosted quick onboar Building or customizing the guest OS itself? Follow the [guest-OS build guide](./docs/building-guest-os.md). +Developing without TEE hardware? Use a development image with +[no-TEE mode and swtpm](./docs/development-without-tee.md). + ## Architecture ![Architecture](./docs/assets/arch.png) @@ -87,15 +90,18 @@ docs/ User and operator documentation tools/ Standalone development and security tools ``` -The currently implemented OS backend is Yocto under `os/yocto/`. Shared rootfs -payload and release assembly stay outside that backend so another builder can -be added later without duplicating them. See [`os/README.md`](./os/README.md). +The default and recommended OS backend is mkosi under `os/mkosi/`. The Yocto +backend under `os/yocto/` is deprecated and kept only to rebuild existing +images; do not use it for new work. Shared rootfs payload and release assembly +stay outside the backends so they are not duplicated. See +[`os/README.md`](./os/README.md). Scripts follow the same ownership boundaries: component-specific helpers stay beside their component under `dstack/`; files installed into every guest live in `os/common/rootfs/`; backend-neutral image tooling lives in `os/image/`; -Yocto-only helpers live in `os/yocto/scripts/`; and repository-wide standalone -utilities live in `tools/`. +mkosi-only helpers live in `os/mkosi/scripts/` (deprecated Yocto-only helpers +in `os/yocto/scripts/`); and repository-wide standalone utilities live in +`tools/`. Your container runs inside a Confidential VM, such as Intel TDX or AMD SEV-SNP, with optional GPU isolation via NVIDIA Confidential Computing. The CPU TEE protects application logic; the GPU TEE protects model weights and inference data. @@ -152,6 +158,8 @@ Apps communicate with the guest agent via HTTP over `/var/run/dstack.sock`. Use - [Gateway](./docs/dstack-gateway.md) - Gateway configuration **Reference** +- [Guest Agent API v1](./docs/guest-api-v1.md) - Key derivation, signature chains, and the versioned guest API +- [Guest Agent API v0](./docs/guest-api-v0.md) - The frozen legacy surface, for clients that still speak the unversioned API - [App Compose Format](./docs/normalized-app-compose.md) - Compose file specification - [Intel TDX Attestation](./docs/attestation-tdx.md) - Measurement and runtime-event verification - [Native TEE Interfaces](./docs/native-tee-interfaces.md) - Advanced compatibility with Linux TEE devices and configfs-tsm @@ -218,7 +226,7 @@ Yes. dstack runs on supported TEE-capable servers, including Intel TDX-capable h
How do users verify my deployment? -Your app exposes attestation quotes via the SDK. Users verify these quotes using [dstack-verifier](https://github.com/Dstack-TEE/dstack/tree/master/dstack/verifier), [dcap-qvl](https://github.com/Phala-Network/dcap-qvl), or the [Trust Center](https://trust.phala.com). See the [verification guide](./docs/verification.md) for details. +Your app exposes attestation quotes via the SDK. Users verify these quotes using [dstack-verifier](https://github.com/Dstack-TEE/dstack/tree/next/dstack/verifier), [dcap-qvl](https://github.com/Phala-Network/dcap-qvl), or the [Trust Center](https://trust.phala.com). See the [verification guide](./docs/verification.md) for details.
diff --git a/REUSE.toml b/REUSE.toml index 52dd38741..68c074019 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -48,7 +48,7 @@ path = [ "sdk/go/go.sum", "sdk/go/ratls/go.sum", "dstack/kms/dstack-app/builder/shared/builder-pinned-packages.txt", - "dstack/kms/dstack-app/builder/shared/qemu-pinned-packages.txt", + "dstack/kms/dstack-app/builder/shared/pinned-packages.txt", "dstack/gateway/dstack-app/builder/shared/builder-pinned-packages.txt", "dstack/gateway/dstack-app/builder/shared/pinned-packages.txt", ] @@ -59,7 +59,6 @@ SPDX-License-Identifier = "Apache-2.0" path = [ "dstack/gateway/templates/wg.conf", "dstack/guest-agent/templates/metrics.tpl", - "dstack/local-key-provider/build/sgx_default_qcnl.conf", ] SPDX-FileCopyrightText = "Copyright (c) 2024-2025 The Project Contributors" SPDX-License-Identifier = "Apache-2.0" @@ -83,7 +82,11 @@ SPDX-FileCopyrightText = "Copyright (c) 2024-2025 The Project Contributors" SPDX-License-Identifier = "Apache-2.0" [[annotations]] -path = "os/common/rootfs/**" +path = [ + "os/common/nvidia/**", + "os/common/rootfs/**", + "os/common/scripts/**", +] SPDX-FileCopyrightText = "Copyright (c) 2024-2025 The Project Contributors" SPDX-License-Identifier = "Apache-2.0" @@ -125,6 +128,16 @@ path = "os/yocto/**" SPDX-FileCopyrightText = "Copyright (c) Hashforest Technology LLC" SPDX-License-Identifier = "Apache-2.0" +# Experimental Debian/mkosi guest OS backend. Supplies the copyright for the +# formats that cannot carry a header; everything that can now does. It is +# deliberately not precedence = "override", so a patch backported from an +# upstream project keeps that project's license instead of being relabelled +# Apache-2.0. +[[annotations]] +path = "os/mkosi/**" +SPDX-FileCopyrightText = "© 2026 Phala Network " +SPDX-License-Identifier = "Apache-2.0" + # Legacy cross-component tools kept outside the Yocto backend. [[annotations]] @@ -172,6 +185,13 @@ SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "Apache-2.0" precedence = "override" +# ktls 6.0.2 plus rustls/ktls#70; see dstack/vendor/README.md. +[[annotations]] +path = "dstack/vendor/ktls/**" +SPDX-FileCopyrightText = "Copyright (c) 2022 Amos Wenger " +SPDX-License-Identifier = "MIT OR Apache-2.0" +precedence = "override" + # Generated files [[annotations]] @@ -242,3 +262,8 @@ path = [ ] SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "CC0-1.0" + +[[annotations]] +path = "dstack/crates/qemu-acpi/fixtures/*.bin" +SPDX-FileCopyrightText = "NONE" +SPDX-License-Identifier = "CC0-1.0" diff --git a/docs/amd-sev-snp.md b/docs/amd-sev-snp.md index a1320ce98..d2ebe8139 100644 --- a/docs/amd-sev-snp.md +++ b/docs/amd-sev-snp.md @@ -82,6 +82,15 @@ Verification is fail-closed and includes: 5. the unified OS image identity, `sha256(sha256sum.txt)`, which must match `digest.txt` and the SNP measurement document. +For a GPU VM, the optional `MrConfigV3.gpu_policy_hash` field contains +`SHA-256(JCS(requirements.gpu_policy))`. The signed SNP report binds the exact +MrConfigV3 document through `HOST_DATA`, so a verifier can validate the report +and document binding and then compare this field with the expected GPU policy +digest. If the field is absent, this optional check is not asserted. This binds +the GPU policy, but not the later `gpu-attestation` runtime event or the +boot-time GPU evidence returned by `/v1/Attest`: the current SEV-SNP path has +no quote-bound runtime measurement register. + The verifier supports the AMD Milan, Genoa, and Turin KDS product families. Bergamo and Siena are handled through AMD's canonical Genoa KDS product path. diff --git a/docs/app-health-checks.md b/docs/app-health-checks.md new file mode 100644 index 000000000..a3f9241c5 --- /dev/null +++ b/docs/app-health-checks.md @@ -0,0 +1,229 @@ +# Application health checks + +The gateway keeps an instance out of its app's load-balancing rotation while the +application is not able to serve. This page describes how an app opts into that +and where the verdict comes from. + +## Why this exists + +A CVM registers with the gateway from `dstack-util setup`, during boot. +`app-compose.service` is ordered *after* `dstack-prepare.service`, so at the +moment of registration the application's containers do not exist yet — the image +may still be pulling. Without a health signal the only thing standing between a +freshly booted instance and a real request is the WireGuard handshake, which +says the tunnel is up and nothing about whether anything is listening behind it. + +## Opting in + +Health gating is off unless the app asks for it, in `requirements`: + +```json +{ + "manifest_version": "3", + "name": "my-app", + "runner": "docker-compose", + "requirements": { + "health_check": true + } +} +``` + +That flag reaches the gateway at registration. An app that does not set it is +never polled and is routed to exactly as it was before this feature existed. + +It lives under `requirements` rather than at the top level for a reason: +`requirements` is rejected by guest images older than `manifest_version` 3, and +unknown fields *inside* it are a hard error. So a deployment that asks for +health gating does not silently land on an image that would ignore it. + +That protection is conditional, and the condition is yours: write +`"manifest_version": "3"` — **the string form**, as above. `AppCompose` itself +is not `deny_unknown_fields`, so an app-compose left at `"manifest_version": 2` +with a `requirements` block is *rejected* by a current guest and *silently +accepted, minus the requirements*, by an older one. The string form is what an +older guest chokes on. + +## Where the verdict comes from + +The guest agent recomputes a verdict every 5 seconds and caches it; the gateway +polls `/prpc/v1/Health`, which only reads that cache. The two cadences are +independent on purpose — a fleet of gateway nodes polling the same instance must +not multiply into that many container-runtime queries inside the CVM, and the +RPC is served on the CVM's publicly reachable listener. + +An app that never set `health_check` has no verdict to read: `/prpc/v1/Health` +answers `healthy: true` to whoever asks, whatever its containers are doing. No +gateway asks, so this only matters to a third party calling the RPC directly — +and there, `true` means "this app opted out of health gating", not "the agent +checked and the app is serving". + +There are two sources, and the app picks one. + +### Default: container healthchecks + +With no `health_status_file`, the agent judges the app's own Compose project: + +```yaml +services: + web: + image: my-app:1.0 + healthcheck: + test: ["CMD-SHELL", "curl -fsS localhost:8080/healthz || exit 1"] + interval: 10s + timeout: 3s + retries: 3 +``` + +Only containers that declare a `healthcheck` are judged. A container without one +is not evidence either way — the app has not said how to test it — so it is +skipped rather than counted as passing. `starting` counts as *not* healthy: that +state is precisely the boot window this exists to keep traffic out of. + +If containers are running but **not one of them declares a healthcheck**, the +instance reports unhealthy rather than healthy. You asked for traffic to be +gated on an answer that nothing can produce, and reporting a pass would leave +you with a feature that is switched on and doing nothing, visible only as the +absence of an effect. Note that this is judged from what the runtime did, not +from your compose file — a `HEALTHCHECK` in a Dockerfile counts, a `healthcheck` +merged in from a YAML anchor counts, and `healthcheck: {disable: true}` does +not, whatever the file appears to say. + +A container that declares a healthcheck and is **not running** is not healthy +either, whatever verdict the runtime last recorded for it. Docker re-marks a +stopped container `unhealthy`; nerdctl leaves the last result in place, so a +container that passed one check and then crashed still reports `healthy` with +`Running: false`. + +Finding **no containers at all** is not healthy either. A runtime that answers +while having nothing to show means Compose has not got as far as creating +anything. + +Only the app's own Compose project is judged, matched on the +`com.docker.compose.project` label. Container state lives on the persistent data +disk and outlives reboots and upgrades, so without that filter one exited +container left behind by an earlier deployment under a different project name +would hold the instance out of rotation permanently. + +The `nerdctl-compose` runner needs **nerdctl >= 2.3.1** for Compose +`healthcheck:` to be honoured; earlier versions parse the field and discard it. +Container presence is still detected on older versions, so the boot-window gate +works regardless. + +### `health_status_file`: the app answers for itself + +For the `bash` runner there are no containers to inspect, and an app may anyway +want to give one whole-application answer rather than have every container +judged separately. Name a file and the agent reads it: + +```json +{ + "requirements": { + "health_check": true, + "health_status_file": "/dstack/health" + } +} +``` + +The file is two lines: + +```text +healthy +1771234567 +``` + +| Line | Meaning | +|---|---| +| 1 | `healthy` or `unhealthy`, case-insensitive | +| 2 | Unix timestamp, in seconds, at which the app wrote the file | + +The timestamp is not decoration. A file older than **60 seconds** counts as +unhealthy, which is what turns a wedged app — still running, no longer updating +anything — into a verdict rather than a stale `healthy` that never expires. +Refresh it at least twice per that window; writing it once at startup is a way +to take yourself out of rotation a minute later. + +Anything else is not healthy: a missing file, a malformed one, an unparseable +timestamp, or a state word the agent does not recognise. Writing it atomically +(write a temporary file in the same directory, then rename) avoids being judged +on a half-written one. + +It must be a **regular file**. A FIFO is refused outright rather than reported +as unhealthy-with-a-reason: opening one with no writer would park a thread of +the agent's blocking pool on every refresh, and those threads are not +reclaimable. + +Symlinks *are* followed, including a symlinked parent directory. The path is +measured into the compose hash but what sits at that path at runtime is not, so +this is a read the app can redirect — which buys it nothing, because it already +writes the file and can therefore already lie about its own health. The read is +bounded and the contents are never quoted back. + +That last point is deliberate: a verdict names the rule that failed — "line 1 is +neither healthy nor unhealthy" — because the report is served to anonymous +callers and the bytes at that path may not be the bytes the app wrote. + +The agent reads the file from the guest rootfs, not from inside any container, +so a container that writes it needs the path bind-mounted in. **Mount the +directory, not the file.** A single-file bind mount pins an inode: a rename +inside the container replaces the container's own directory entry and never +touches the file the agent is reading, so the agent sees the original contents +forever, the timestamp ages past the limit, and the instance goes permanently +unhealthy. With the directory mounted, either write-then-rename or a plain +in-place write works. + +Before the first refresh completes, the app reports **not** healthy. +Registration happens long before the application is up, so "not determined yet" +must not read as a pass. + +## How the gateway uses it + +- Health only gates **app-id** routing. Addressing an instance directly by + instance id is never gated, so an unhealthy instance stays reachable for + investigation. +- **If no instance of an app reports healthy, the gateway routes to all of them + anyway.** Health is inference and can be wrong — a misconfigured probe, an + agent that died, a whole fleet failing for a reason unrelated to the app. + Blackholing an app on that basis is worse than sending traffic to instances + that might be fine. Note the consequence: for a **single-instance** app, or a + fleet that boots together, this rule routes traffic while every instance is + still `unknown`. Health gating shifts traffic away from a bad instance; it + does not hold an app offline until one is good. +- A poll that cannot reach the agent is not immediately a verdict. It takes + `failure_threshold` consecutive failures to demote an instance, so one dropped + packet does not eject it. An agent that *answers* "unhealthy" is believed on + the spot. +- Health is a *per-node observation*. Each gateway node polls for itself and + does not share the result, the same way each node reads its own WireGuard + handshakes. The verdict is not persisted: a gateway restart puts every + instance back at `unknown`, and since that is the whole app at once, the + fail-open above routes to all of them until the first round of polls answers + a few seconds later. What *is* stored is the CVM's declaration -- the + `health_check` field of its `inst/` record -- so a restarted node knows which + instances to poll before any of them re-register. +- Instances whose WireGuard handshake has gone stale are not polled. They are + already excluded from routing, and polling them would spend the round's budget + on CVMs that are gone. + +Operators can also take an instance out of rotation by hand with +`Admin.SetInstanceReady`, independently of health. That one is an instruction +rather than an inference, so it does **not** fail open: gating every instance of +an app means the app refuses new connections. + +Polling is configured on the gateway: + +```toml +[core.proxy.health_check] +enabled = true +interval = "5s" +timeout = "2s" +concurrency = 16 +failure_threshold = 2 +``` + +`enabled = false` restores the behaviour from before health polling existed: +every instance stays eligible regardless of what it reports. Note what that +looks like from the outside: instances that opted in sit at `unknown` forever, +because nothing will ever poll them, and they are *all in rotation*. The gateway +dashboard labels that case `unknown (gating off)` and reports the switch itself +under **This Node → Health Gating**, so the state is not left to be inferred +from a column that means something different when polling is on. diff --git a/docs/attestation-tdx.md b/docs/attestation-tdx.md index f0f433423..14c20bcf5 100644 --- a/docs/attestation-tdx.md +++ b/docs/attestation-tdx.md @@ -25,12 +25,28 @@ The MR register values indicate the following: - RTMR0: OVMF records CVM's virtual hardware setup, including CPU count, memory size, and device configuration. While dstack uses fixed devices, CPU and memory specifications can vary. RTMR0 can be computed from these specifications. - RTMR1: OVMF records the Linux kernel measurement. - RTMR2: Linux kernel records kernel cmdline (including rootfs hash) and initrd measurements. - - RTMR3: initrd records dstack App details, including compose hash, instance id, app id, and key provider. + - RTMR3: initrd records dstack App details, including compose hash, GPU policy and attestation events, instance id, app id, and key provider. MRTD, RTMR0, RTMR1, and RTMR2 can be pre-calculated from the built image (given CPU+RAM specifications). Compare these with the verified quote's MRs to confirm correct base image code execution. RTMR3 differs as it contains runtime information like compose hash and instance id. Verify this by replaying the event log - if the calculated RTMR3 matches the quote's RTMR3, the event log information is valid. Then verify the compose hash, key provider, and other event log details match expectations. +After `compose-hash`, each configured init script produces an ordered +`init-script-hash` event whose payload is the SHA-256 digest of the exact UTF-8 +script bytes. The event order matches the script array order. These events let +an infrastructure provider contribute initialization code approved by +multiple parties and let each party verify its code independently without +reconstructing the complete compose document. + +For a GPU launch, any `init-script-hash` events are followed by +`gpu-policy-hash` and, after successful NVIDIA attestation and policy +evaluation, `gpu-attestation`. The `gpu-policy-hash` payload is +`SHA-256(JCS(requirements.gpu_policy))`, using `{}` when the field is omitted. +The `gpu-attestation` payload is JSON containing the verified device count, +CC/DevTools state, and `evidence_sha256`. + +The guest-agent returns the complete `nvattest` record captured during boot from `/v1/Attest`, when the request sets `include_boottime_gpu_evidence`, so a verifier can fetch the quote and the GPU evidence in one round trip. `AttestResponse.boottime_gpu_evidence` is a list of `GpuEvidenceBundle` (`{vendor, format, evidence}`); the boot record is the bundle whose `vendor` is `nvidia` and whose `format` is `nvidia-nvattest-boottime-json-v1`, and its `evidence` is hex-encoded bytes that decode to the exact UTF-8 `nvattest` output. It is not trustworthy by itself. (`/v1/AttestGpu` runs a *fresh* attestation against a caller nonce and returns bundles tagged `nvidia-nvattest-collect-evidence-json-v1`, a deliberately distinct format that a boot-record verifier does not appraise; its result is not bound to the TD and must not be used as remote evidence; only the boot-time record below is.) After verifying the TDX quote and replaying the event log to RTMR3, hash the *decoded* bundle bytes — `SHA-256(hex_decode(bundle.evidence))`, never the JSON string as returned nor a re-serialized form — and require the result to equal the `gpu-attestation` event's `evidence_sha256`. See [GPU Security for AI Workloads](./security/security-model.md#gpu-security-for-ai-workloads) for the event schema, ordering, Rego example, and platform differences. + ### 2.2. Determining expected MRs MRTD, RTMR0, RTMR1, and RTMR2 correspond to the image. dstack OS builds all related software from source. Build the exact image revision you intend to verify. See @@ -44,7 +60,8 @@ git checkout make os-image ``` -The resulting `dstack-.tar.gz` contains: +The resulting `os/mkosi/repro-build/build/out/prod/dstack-.tar.gz` +contains: - ovmf.fd: virtual firmware - bzImage: kernel image @@ -67,7 +84,7 @@ To verify dstack App data trustworthiness: - Review source code for correctness and safety. - Build image from source. -- Calculate MRTD, RTMR0, RTMR1, and RTMR2 values using [dstack-mr](https://github.com/Dstack-TEE/dstack/tree/master/dstack/dstack-mr). +- Calculate MRTD, RTMR0, RTMR1, and RTMR2 values using [dstack-mr](https://github.com/Dstack-TEE/dstack/tree/next/dstack/dstack-mr). - Verify quote measurements: - Confirm MRTD, RTMR0, RTMR1, and RTMR2 match pre-calculated values. - Verify RTMR3 matches the event log replay result. diff --git a/docs/aws-attested-instance-security-evaluation.md b/docs/aws-attested-instance-security-evaluation.md index d772f3022..4f3ca9682 100644 --- a/docs/aws-attested-instance-security-evaluation.md +++ b/docs/aws-attested-instance-security-evaluation.md @@ -35,7 +35,7 @@ NitroTPM path meets them today. | P1 | Verifiable platform root of trust | TDX/SNP/Nitro quote verification against vendor root; debug rejected; TCB surfaced | NitroTPM Attestation Documents are verified against the AWS Nitro Attestation PKI, including document timestamp sanity. AWS exposes no TDX/SNP-style TCB advisory field, so a verified attestation is normalized to `tcbStatus = "UpToDate"` and passes the standard authorization gate unchanged. | | P2 | Reproducible or independently computable base image measurement | meta-dstack rebuild plus `dstack-mr` computes `MRTD`/`RTMR0-2` | The unified `os/build.sh` flow emits the AWS image archive with `sha256sum.txt`, `digest.txt`, and `measurement.aws.cbor`; its output directory also contains the reference-PCR side-car `aws-pcrs.json`. `os_image_hash = sha256(sha256sum.txt)` is the same identity used on all platforms; the verifier recomputes it from the downloaded image directory (`dstack/verifier/src/verification.rs`). The hardening audit script `os/yocto/tools/aws/audit-aws-ec2-image-hardening.sh` checks the image for operator mutation channels. | | P3 | Boot command line and root filesystem integrity are measured | `RTMR1/2`, rootfs hash, dm-verity, measured initrd/cmdline | The UKI commits kernel, initrd, and embedded cmdline into `PCR4`; the rootfs is dm-verity-protected. `VmConfig.aws_measurement` is required and must bind `boot_pcr_digest = sha256(PCR4||PCR7||PCR12)` to the attested PCRs, so `PCR12` (external cmdline) is always part of the bound digest — a missing-PCR12 bypass is not expressible. Enforced in guest quote generation (`dstack/dstack-attest/src/attestation.rs`), `verify_os_image_hash_for_aws_nitro_tpm` (`dstack/verifier/src/verification.rs`), and the KMS pipeline via the same verifier check. | -| P4 | Runtime application identity is cryptographically bound | RTMR3 `compose-hash`, `app-id`, `instance-id`, `key-provider`; event log replay | SHA384 `PCR14` event-log replay is the authoritative binding (RTMR3 analogue; non-resettable). Launch events: `system-preparing`, `app-id`, `compose-hash`, `instance-id`, `boot-mr-done`, `key-provider`, `storage-fs`, `system-ready` (`dstack/dstack-util/src/system_setup.rs`). `dstack-attest`, `dstack-verifier`, and KMS reject missing/mismatched PCR14 and bad replay. Optionally, the guest extends the raw `MrConfig` V2 `config_id` into `PCR8` once (`PCR8 = sha384(0^48 || config_id)`) so a lightweight third-party verifier can check compose hash + key provider without event-log replay; dstack's own verifier and KMS do not check PCR8. | +| P4 | Runtime application identity is cryptographically bound | RTMR3 `compose-hash`, `app-id`, `instance-id`, `key-provider`; event log replay | SHA384 `PCR14` event-log replay is the authoritative binding (RTMR3 analogue; non-resettable). Launch events: `system-preparing`, `app-id`, `compose-hash`, zero or more ordered `init-script-hash` events, `instance-id`, `boot-mr-done`, `key-provider`, `storage-fs`, `system-ready` (`dstack/dstack-util/src/system_setup.rs`). GPU launches also include `gpu-policy-hash` and `gpu-attestation` before `instance-id`. `dstack-attest`, `dstack-verifier`, and KMS reject missing/mismatched PCR14 and bad replay. Optionally, the guest extends the raw `MrConfig` V2 `config_id` into `PCR8` once (`PCR8 = sha384(0^48 || config_id)`) so a lightweight third-party verifier can check compose hash + key provider without event-log replay; dstack's own verifier and KMS do not check PCR8. | | P5 | Challenge/liveness and caller key binding | `report_data` challenge or RA-TLS public key hash in quote | RA-TLS binds `report_data` to the TLS certificate public key. KMS key release is bound to the live RA-TLS handshake; external `/verify` callers supply and check their own `report_data` challenge. The low-level NitroTPM document verifier also rejects stale or far-future document timestamps. | | P6 | Secret release only to attested code | dstack KMS verifies attestation, checks auth policy, derives per-app keys | dstack KMS verifies the NitroTPM attestation, runs the same `verify_os_image_hash_for_aws_nitro_tpm` binding check as the verifier, builds `BootInfo` from verified boot PCRs plus PCR14 launch events, and checks auth policy before deriving app keys (`dstack/kms/src/main_service.rs`). AWS NitroTPM key release is gated behind the opt-in `aws_nitro_tpm_key_release` flag (default false in `kms.toml`). | | P7 | Key-release policy is not controlled by the untrusted account admin | KMS runs inside TEE; policy from auth API/contracts; KMS identity measured | Satisfied with dstack KMS or another verifiable secret authority outside the untrusted AWS account admin's control. A NitroTPM-backed dstack KMS keeps root material out of account-admin snapshots and clones. The policy backend (auth-simple in a trusted control plane, or on-chain `DstackKms`/`DstackApp`) must be outside the workload account admin's control. Same-account AWS KMS fails this property if the admin can change key policy, create grants, or call secret-bearing operations through a policy they control. | diff --git a/docs/aws-ec2-production-verifier-runbook.md b/docs/aws-ec2-production-verifier-runbook.md index c5a5e35d3..d43fc3719 100644 --- a/docs/aws-ec2-production-verifier-runbook.md +++ b/docs/aws-ec2-production-verifier-runbook.md @@ -11,8 +11,8 @@ and endpoint identity. ## Inputs - The release image package for the exact release candidate, produced by the - unified build entrypoint `os/build.sh` (the `--uki.tar.gz` - dist archive). It contains `disk.raw`, `sha256sum.txt`, `digest.txt`, and + guest-OS build (`make os-image`; see `docs/building-guest-os.md`) as the + `--uki.tar.gz` archive. It contains `disk.raw`, `sha256sum.txt`, `digest.txt`, and `measurement.{gcp,aws}.cbor`. The build output also contains an `aws-pcrs.json` side-car, but that file is not part of the archive. - The dstack monorepo sources pinned at the exact release revision, for the @@ -27,14 +27,14 @@ and endpoint identity. The release evidence is the image package itself: every measured file is listed in `sha256sum.txt`, and the unified image identity is `os_image_hash = sha256(sha256sum.txt)` (also recorded as `digest.txt`). -There is no separately generated release manifest; the same `os/build.sh` -flow produces both the image and its evidence. +There is no separately generated release manifest; the same build produces +both the image and its evidence. Rebuild from clean, pinned sources and require a byte-identical result: ```bash git status --porcelain # must be empty at the pinned release revision -./os/build.sh # reproducible Yocto backend; emits images/--uki.tar.gz +make os-image # reproducible mkosi backend; emits os/mkosi/repro-build/build/out/prod/--uki.tar.gz ``` Compare the rebuilt package against the published one. `sha256sum.txt` lists @@ -54,9 +54,9 @@ expected=$(sha256sum published/sha256sum.txt | awk '{print $1}') test "$expected" = "$(cat published/digest.txt)" ``` -BitBake enforces per-recipe input checksums during the rebuild; for full -supply-chain independence, mirror the Yocto `downloads/` input cache by -content hash. +The mkosi backend pins its inputs to an immutable Debian snapshot, SHA-256 +checked archives, and fixed Git revisions (`os/mkosi/versions.env`); for full +supply-chain independence, mirror those inputs by content hash. Run the hardening audit against the release kernel config and rootfs; it must exit zero with `failures=0`: @@ -181,7 +181,7 @@ jq -e \ .is_valid == true and .details.quote_verified == true and .details.report_data == $expected_report_data and - .details.boot_info.attestationMode == "dstack-aws-nitro-tpm" and + .details.boot_info.teeVariant == "dstack-aws-nitro-tpm" and .details.boot_info.tcbStatus == "UpToDate" and .details.boot_info.osImageHash == $expected_os_image_hash ' request.json.verification.json @@ -209,6 +209,18 @@ For AWS NitroTPM, the policy must require: `instance-id`, `key-provider`); - a `report_data` challenge for external verifier flows that need liveness. +For GPU workloads, PCR14 also contains `gpu-policy-hash` immediately after +`compose-hash`. Its 32-byte payload must equal +`SHA-256(JCS(requirements.gpu_policy))`, using `{}` when the policy is omitted. +Because the verifier replays the event chain against the signed NitroTPM +Attestation Document, this validates `gpu_policy_hash` on AWS. A successful GPU +launch also adds `gpu-attestation`; after PCR14 replay, its `evidence_sha256` +can be compared with `SHA-256(hex_decode(bundle.evidence))`, where `bundle` is +the `AttestResponse.boottime_gpu_evidence` entry whose `format` is +`nvidia-nvattest-boottime-json-v1`, returned by `/v1/Attest` when the request +sets `include_boottime_gpu_evidence`. Hash the decoded bytes exactly as +returned, not a re-serialized form. + The guest also extends a `MrConfig` V2 **config commitment** into **PCR8** (`PCR8 = sha384(0^48 || config_id)`). This is **optional** and exists only to let a lightweight third-party verifier confirm `composeHash` + key-provider @@ -218,9 +230,9 @@ key_provider_id })` and compare it to PCR8. dstack's own verifier and KMS do **not** check PCR8 — they rely on PCR14 replay — so it is not part of the required policy above. -The `attestationMode` field is carried in the verified boot info for +The `teeVariant` field is carried in the verified boot info for observability; a relying party may additionally assert -`attestationMode == "dstack-aws-nitro-tpm"` against the verifier output, but the +`teeVariant == "dstack-aws-nitro-tpm"` against the verifier output, but the authorization contract itself keys on the standard fields above. Same-account AWS KMS is not a trusted secret authority in this threat model if @@ -253,7 +265,7 @@ Require the generated `endpoint-cert.pem.ratls-verification.json` to contain: ```bash jq -e ' .is_valid == true and - .details.attestation_mode == "dstack-aws-nitro-tpm" and + .details.tee_variant == "dstack-aws-nitro-tpm" and .details.app_info.os_image_hash_verified == true ' endpoint-cert.pem.ratls-verification.json ``` @@ -272,7 +284,7 @@ An AWS EC2 NitroTPM deployment can be promoted only when: 1. a reproducible rebuild from the clean, pinned release sources yields a byte-identical `sha256sum.txt` (and therefore the same `os_image_hash`); -2. Yocto input content is mirrored or otherwise available by content hash; +2. build input content is mirrored or otherwise available by content hash; 3. independently recomputed AWS PCRs and the UKI AuthentiCode hash match the published build-output `aws-pcrs.json` side-car (when provided) and the `boot_pcr_digest` in `measurement.aws.cbor`; diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index a96d5c9dc..d319a7827 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -4,7 +4,7 @@ By default, dstack-vmm uses **user** networking (QEMU's built-in SLIRP stack, no ## When to use bridge networking -- High connection concurrency (passt becomes CPU-bound at ~25K+ concurrent connections) +- High connection concurrency (user-mode networking becomes CPU-bound at ~25K+ concurrent connections) - Workloads that need full L2 network access - Environments where VMs need to be directly reachable on the LAN @@ -21,11 +21,11 @@ bridge = "virbr0" ### Per-VM override Individual VMs can override the global networking mode via: -- **CLI**: `vmm-cli.py deploy --net bridge` or `--net passt` +- **CLI**: `vmm-cli.py deploy --net bridge`, `--net user`, or `--net macvtap` - **Web UI**: Networking dropdown in the deploy dialog - **API**: `networking: { mode: "bridge" }` in `VmConfiguration` -Only the mode is per-VM; the bridge interface name always comes from the global config. +The bridge interface name comes from the global config unless the node lists it in `cvm.allowed_bridges`. VMs may also override the vhost and queue settings — see [network-data-plane.md](network-data-plane.md). ## Host setup @@ -90,13 +90,6 @@ sudo sysctl -p /etc/sysctl.d/99-dstack-bridge.conf sudo apt install -y dnsmasq ``` -Install the DHCP notification script (notifies VMM when a VM gets an IP so port forwarding can be established): - -```bash -sudo cp dstack/scripts/dhcp-notify.sh /usr/local/bin/dhcp-notify.sh -sudo chmod +x /usr/local/bin/dhcp-notify.sh -``` - Create dnsmasq config: ```ini @@ -106,11 +99,8 @@ bind-interfaces dhcp-range=10.0.100.10,10.0.100.254,255.255.255.0,12h dhcp-option=option:router,10.0.100.1 dhcp-option=option:dns-server,8.8.8.8,1.1.1.1 -dhcp-script=/usr/local/bin/dhcp-notify.sh ``` -The `dhcp-script` option tells dnsmasq to call the notification script on every lease event. The script sends the MAC and IP to VMM's `ReportDhcpLease` RPC, which triggers automatic port forwarding for the VM. - ```bash sudo systemctl restart dnsmasq ``` @@ -153,29 +143,36 @@ mode = "bridge" bridge = "dstack-br0" ``` -### QEMU bridge helper setup (required for both options) +### netd is required -The bridge helper allows QEMU to create and attach TAP devices without VMM needing root privileges. +Bridge networking needs `netd`, the privileged helper that owns every host +interface a bridge or macvtap NIC uses. It is the same binary: ```bash -# Allow QEMU to use the bridge -sudo mkdir -p /etc/qemu -echo "allow virbr0" | sudo tee /etc/qemu/bridge.conf -# Or for manual bridge: echo "allow dstack-br0" | sudo tee /etc/qemu/bridge.conf - -# Set setuid on bridge helper -sudo chmod u+s /usr/lib/qemu/qemu-bridge-helper +sudo dstack-vmm --config vmm.toml netd ``` +Nothing else on the node needs `CAP_NET_ADMIN`: the VMM itself still runs +unprivileged, and `netd` holds the privilege behind a Unix socket whose +filesystem permissions authorize callers. + +This used to be conditional — `netd` built the TAP when libvirt filtering was on +or when the NIC wanted more than one queue pair, and otherwise QEMU's setuid +`qemu-bridge-helper` did. Two owners meant two answers to the same questions: +which netdev QEMU gets, whether vhost is really on, and what a bridge NIC's TAP +is built with. So a bridge NIC's host interface has one owner now, on every +node. + +`qemu-bridge-helper` is no longer used, and `/etc/qemu/bridge.conf` no longer +needs an `allow` line for the bridge. + ## How it works -- VMM passes `-netdev bridge,id=net0,br=` to QEMU -- QEMU's bridge helper (setuid) creates a TAP device and attaches it to the bridge +- `netd` creates a persistent TAP, attaches it to the bridge, binds the nwfilter if the node filters, and the VMM passes `-netdev tap,id=net0,ifname=,...` - Guest MAC address is derived from SHA256 of the VM ID, with an optional configurable prefix (stable across restarts for DHCP IP consistency) -- The host DHCP server (dnsmasq) assigns an IP and calls `dhcp-notify.sh`, which notifies VMM via the `ReportDhcpLease` RPC -- VMM matches the MAC address to identify the VM and establishes port forwarding rules -- When QEMU exits, the TAP device is automatically destroyed -- VMM does not need root or `CAP_NET_ADMIN` +- The host DHCP server (dnsmasq) assigns an IP to the VM +- The TAP outlives QEMU and is deleted when the VMM tears the VM's networking down +- The VMM process needs neither root nor `CAP_NET_ADMIN`; `netd` holds that privilege in a separate service ### MAC address prefix @@ -203,15 +200,54 @@ The remaining bytes are derived from the VM ID hash. The prefix applies to all n - Docker's nftables chains (`DOCKER-FORWARD`) run before libvirt's but do not block virbr0 traffic - Use `setup-bridge.sh check --bridge ` to diagnose missing rules +### Which NIC a port mapping uses + +A port mapping says which NIC its traffic enters through: + +```bash +vmm-cli.py deploy ... --port udp:0.0.0.0:7483:51820@0 --port tcp:127.0.0.1:7484:8001@0 +``` + +Leave `@` off and the VMM picks the first user-mode NIC — where QEMU's +`hostfwd=` entries have always gone. If the VM has no user-mode NIC, the mapping +has no publishing backend and the launch log names it as stranded. A single-NIC +user-mode VM never needs the suffix. + +With several NICs the choice used to be made silently, and not always the way an +operator would have. A bridge NIC for external traffic beside a user-mode NIC for +management — the topology multi-NIC was added for — put every published port on +the *management* NIC: the traffic reached the guest, but over slirp, bypassing +whatever the bridge NIC's nwfilter was there to enforce and hiding the client's +address behind the slirp gateway. A second user-mode NIC could never publish +anything at all, because only the first was ever selected. + +A mapping resolves to at most one NIC. The only backend that can carry it is +QEMU user networking through `hostfwd=`; `netd` builds bridge interfaces but +does not publish host ports. + +### Which ports a bridge NIC can publish + +QEMU publishes a port with `hostfwd=` on a user-mode NIC, and that is the only +mechanism this host has. **The `netd` in this repository builds interfaces; it +does not forward host ports**, so a bridge NIC cannot carry a port mapping. + +`--port …@` therefore only ever names a user-mode NIC. Pinning to a bridge, +macvtap or custom NIC is refused at deployment, where the caller is there to be +told. An unpinned mapping goes to the first user-mode NIC; a VM that has none is +not refused — it may have been deployed before this — but every mapping it +strands is named in the launch log. + ### Mixing networking modes -Bridge and passt VMs can coexist. Set the global default in `vmm.toml` and override per-VM as needed: +Bridge and user-mode VMs can coexist. Set the global default in `vmm.toml` and override per-VM as needed: ```bash -# Global default is bridge, but deploy this VM with passt -vmm-cli.py deploy --name my-vm --image dstack-0.5.6 --compose app.yaml --net passt +# Global default is bridge, but deploy this VM with user networking +vmm-cli.py deploy --name my-vm --image dstack-0.5.6 --compose app.yaml --net user ``` -### vhost-net and TDX +### vhost-net and multiqueue + +Bridge NICs can run on the host kernel's vhost-net data plane and expose several virtio-net queue pairs. Both are off by default and enabled per node or per VM — see [network-data-plane.md](network-data-plane.md) for the knobs, the enablement checklist, the mode support matrix, and how to pick a queue count. -vhost-net (kernel data plane offload for virtio-net) is **not enabled** for bridge mode. TDX encrypts guest memory, which prevents the host kernel from performing DMA-based packet offload. The default QEMU userspace virtio backend is used instead. +vhost-net works in a TDX guest: the virtio rings and buffers live in shared, unencrypted memory so that a host-side backend can reach them, which is the same mechanism `vhost-vsock-pci` has always relied on. diff --git a/docs/building-guest-os.md b/docs/building-guest-os.md index f783ac327..2480b7cc1 100644 --- a/docs/building-guest-os.md +++ b/docs/building-guest-os.md @@ -6,6 +6,16 @@ image. You do **not** need to build an image for normal self-hosted onboarding: `dstackup install` downloads and verifies a published guest-OS release by default. +> [!IMPORTANT] +> **mkosi is the default and recommended OS backend.** The Yocto backend under +> `os/yocto/` is **deprecated**. It remains only so existing Yocto images can +> still be rebuilt. Do not use it for new images, new features, or release +> work. See [Deprecated: Yocto backend](#deprecated-yocto-backend). +> +> The mkosi build still reads some patches, units, and scripts from +> `os/yocto/`, so that directory must not be deleted yet; see +> [`../os/README.md`](../os/README.md). + ## What the build produces The default `prod` build produces: @@ -16,19 +26,26 @@ The default `prod` build produces: - dm-verity rootfs data, launch-measurement material, checksums, and the unified `digest.txt` OS identity. -Yocto is currently the only implemented OS backend. Backend-independent rootfs -payload, artifact contract, measurement, and release packaging live outside -`os/yocto/`; see [`../os/README.md`](../os/README.md). +The mkosi backend lives in `os/mkosi/`. Backend-independent rootfs payload, +artifact contract, measurement, and release packaging live outside the backend; +see [`../os/README.md`](../os/README.md). Backend internals, the +reproducibility model, and the component cache are described in +[`../os/mkosi/README.md`](../os/mkosi/README.md). ## Prerequisites Use an x86-64 Linux host with: - Git; -- Docker Engine, usable by the current user; -- outbound HTTPS access for Git, Yocto source archives, and Rust crates; -- substantial free disk space for Yocto downloads, work directories, and - shared-state cache. +- Docker Engine, usable by the current user, that can run `--privileged` + containers (mkosi needs loop devices, device-mapper, and mounts); +- outbound HTTPS access for the pinned Debian snapshot, upstream source + archives, and Rust crates; +- tens of gigabytes of free disk space on a regular filesystem (not overlayfs) + for the build directory. + +No Rust, Go, C, or C++ toolchain is needed on the host; every compiler is pinned +and runs inside mkosi's build overlay. TEE hardware is not required to build the image. It is required only when you boot and attest the resulting image on the corresponding platform. @@ -51,42 +68,37 @@ cd dstack make os-image ``` -`make os-image` initializes only the eight Yocto dependency submodules and -runs one complete production image build in the pinned Ubuntu builder -container. It is equivalent to: +`make os-image` runs one complete, cold production image build in a container +that pins mkosi and the host tools it drives. It needs no submodules and is +equivalent to: ```bash -git submodule update --init --depth 1 -- \ - os/yocto/deps/bitbake \ - os/yocto/deps/openembedded-core \ - os/yocto/deps/meta-yocto \ - os/yocto/deps/meta-confidential-compute \ - os/yocto/deps/meta-virtualization \ - os/yocto/deps/meta-openembedded \ - os/yocto/deps/meta-rust-bin \ - os/yocto/deps/meta-security - -cd os/yocto/repro-build -./repro-build.sh -n +./os/mkosi/repro-build/repro-build.sh ``` -The first build downloads and compiles the complete Yocto toolchain and guest -userspace, so it is much slower than an incremental rebuild. The `-n` option -means “build once”; it does not skip BitBake or image assembly. +The build refuses to run from a dirty worktree, because the recorded source +revision would not describe the compiled sources. Commit or stash changes +first. (Native `os/mkosi/build.sh` builds also accept `DSTACK_ALLOW_DIRTY=1`, +which records the revision as `-modified`; the containerized build does not +forward it.) + +A cold production build takes roughly 30–45 minutes on a 16-job host, +depending mostly on network speed. ## Outputs -Release archives are written under: +Release artifacts are written under: ```text -os/yocto/repro-build/dist/ +os/mkosi/repro-build/build/out/prod/ +├── dstack-/ ├── dstack-.tar.gz -├── dstack--uki.tar.gz -└── reproduce.sh +└── dstack--uki.tar.gz ``` -`reproduce.sh` is emitted when the source tree is clean. The unpacked build -tree and caches remain under `os/yocto/repro-build/build-a/`. +To use a different build directory, run +`./os/mkosi/repro-build/repro-build.sh -o DIR`. When the build finishes it +prints the `os_image_hash` and the SHA-256 of both archives. The bare-metal archive includes the kernel, initramfs, OVMF firmware, partitioned dm-verity rootfs, platform measurement CBOR files, @@ -97,7 +109,7 @@ Inspect and verify an archive with: ```bash mkdir -p /tmp/dstack-image -tar -xzf os/yocto/repro-build/dist/dstack-.tar.gz \ +tar -xzf os/mkosi/repro-build/build/out/prod/dstack-.tar.gz \ -C /tmp/dstack-image cd /tmp/dstack-image/dstack- sha256sum -c sha256sum.txt @@ -109,112 +121,102 @@ test "$(sha256sum sha256sum.txt | awk '{print $1}')" = "$(cat digest.txt)" Production is the default. To build both variants once: ```bash -cd os/yocto/repro-build -RELEASE_FLAVORS="prod dev" ./repro-build.sh -n +FLAVORS="prod dev" ./os/mkosi/repro-build/repro-build.sh ``` -The development archive is named `dstack-dev-.tar.gz` and records -`"is_dev": true` in `metadata.json`. +The development archive is written to `out/dev/dstack-dev-.tar.gz` +and records `"is_dev": true` in `metadata.json`. ## Check reproducibility -For a release candidate, omit `-n`: +For a release candidate: ```bash make os-repro-check ``` -This builds independent `build-a` and `build-b` trees and compares the -release-relevant output allowlist. It takes roughly twice the resources of a -single build. Remove both ignored build trees if you specifically need a -from-scratch comparison: - -```bash -rm -rf os/yocto/repro-build/build-a \ - os/yocto/repro-build/build-b \ - os/yocto/repro-build/dist -make os-repro-check -``` +This performs two cold production builds in different build paths and with +different job counts, then compares both release archives byte for byte. It +takes roughly twice the resources of a single build. The first leg's artifacts +remain under `os/mkosi/repro-build/build/a/`. ## Incremental backend development -The reproducible wrapper is the recommended release path. On a host with the -packages listed in `os/yocto/repro-build/Dockerfile.repro`, the generic -backend entrypoint can also be used directly from the repository root: +The containerized build is the recommended release path. On a host with the +pinned mkosi version (see `MKOSI_VERSION` in `os/mkosi/versions.env`) and the +packages reported by `mkosi --directory os/mkosi dependencies`, and with root +privileges (or a working user namespace), the backend can be driven directly: ```bash -./os/build.sh \ - --backend yocto \ - --flavors prod \ - --build-dir "$PWD/os/yocto/bb-build" +./os/mkosi/build.sh lint # static contract, seconds +./os/mkosi/build.sh image "$PWD/os/mkosi/build" # cached iteration build +./os/mkosi/build.sh --no-cache image "$PWD/os/mkosi/build" # cold build +./os/build.sh --flavors "prod dev" --build-dir "$PWD/os/mkosi/build" ``` -This keeps the native BitBake cache in `os/yocto/bb-build/` and writes -assembled images under the repository-root `images/` directory. Build both -flavors with `--flavors "prod dev"`. +A native `image` build reuses a component-output cache by default and skips the +release tarballs; `disk.raw`, the measurements, and `metadata.json` are still +produced. Pass `--archive` to get the tarballs from a cached build, or +`--no-cache` for a release-equivalent cold build. See +[`../os/mkosi/README.md`](../os/mkosi/README.md) for cache details. -The generic entrypoint dispatches to `os//build.sh`. A future -backend such as mkosi can implement the same artifact-manifest contract without -changing the common assembler or release consumers. +The generic entrypoint `os/build.sh` dispatches to `os//build.sh` and +defaults to `mkosi`. ## Troubleshooting -### A dependency directory is empty - -Run: - -```bash -make os-deps -git submodule status -- os/yocto/deps -``` - -Every listed dependency should start with a space, not `-`. - ### Docker permission is denied Ensure `docker version` works as the same non-root user that owns the -checkout. Do not run only part of the build as root; mixed ownership in -`build-a/` makes incremental builds difficult to repair. +checkout. The build container runs privileged and hands ownership of the build +directory back to the calling user when it exits. -### A fetch task fails +### The workspace is on overlayfs -Yocto fetches many upstream sources. Preserve `build-a/`, confirm outbound -network and DNS access, then rerun `make os-image`; completed downloads and -tasks are reused. +mkosi assembles its build root as an overlayfs, which cannot be stacked on +another overlayfs. Choose a build directory on a regular filesystem with +`./os/mkosi/repro-build/repro-build.sh -o DIR`. -### `docker-compose do_fetch` repeatedly shows 0–100% +### A fetch fails -This is not one archive being downloaded in a loop. Docker Compose has hundreds -of independently checksummed Go-module sources, while BitBake's terminal -percentage describes only the current source URL. The percentage therefore -returns to zero for every module even though the task timer and PID stay the -same. - -Let the first fetch finish. If it is interrupted, rerun the same command; -completed files have `.done` markers in the build directory's `downloads/` -cache and are not downloaded again. To confirm which URL is currently being -fetched during a native `make os` build, inspect the latest task log: - -```bash -find os/yocto/bb-build/tmp-mc-* -path '*docker-compose/*/temp/log.do_fetch' \ - -print -exec tail -n 5 {} \; -``` +Confirm outbound network and DNS access, then rerun the same command. Package +downloads come from an immutable Debian snapshot, so a retry fetches identical +content. ### The disk fills up -The largest disposable directories are: +The disposable build directories are: ```text -os/yocto/repro-build/build-a/ -os/yocto/repro-build/build-b/ -os/yocto/bb-build/ +os/mkosi/repro-build/build/ +os/mkosi/build/ +~/.cache/dstack/mkosi-dev/ # native component cache ``` -They are ignored by Git and can be removed when no build is running. Keep -`dist/` separately if you need the release archives. - -### `reproduce.sh` is missing - -The image archives are still valid. The wrapper intentionally skips generating -`reproduce.sh` when `git status --porcelain` reports a dirty source tree, -because that script can reproduce only committed source revisions. +They are ignored by Git (or live outside the checkout) and can be removed when +no build is running. Copy the release archives elsewhere first if you need +them. + +## Deprecated: Yocto backend + +> [!WARNING] +> The Yocto backend is deprecated and will be removed. Do not use it for new +> images or releases, and do not add features to it. Every build entrypoint +> prints a deprecation warning. Files under `os/yocto/` that the mkosi build +> still reads remain live inputs of the default image until they are moved. + +The Yocto backend is kept only so existing Yocto-built images can still be +rebuilt and verified. Its entrypoints have moved to explicitly named targets: + +| Deprecated command | Replacement | +|--------------------|-------------| +| `make os-image-yocto` | `make os-image` | +| `make os-repro-check-yocto` | `make os-repro-check` | +| `make os-yocto` / `./os/build.sh --backend yocto` | `make os` / `./os/build.sh` | +| `make os-deps` | not needed; mkosi uses no submodules | + +`make os-image-yocto` initializes the eight Yocto dependency submodules and runs +`os/yocto/repro-build/repro-build.sh -n` in its pinned Ubuntu builder container. +Archives are written to `os/yocto/repro-build/dist/`; use +`RELEASE_FLAVORS="prod dev"` to build both flavors. See +[`../os/yocto/README.md`](../os/yocto/README.md) for the remaining details. diff --git a/docs/certbot-dns-persist-01.md b/docs/certbot-dns-persist-01.md new file mode 100644 index 000000000..c3b264313 --- /dev/null +++ b/docs/certbot-dns-persist-01.md @@ -0,0 +1,240 @@ +# Certificate issuance without a DNS credential (`dns-persist-01`) + +`dns-01` asks certbot to write a fresh `_acme-challenge` TXT record for every +order, so whatever runs certbot holds a DNS API token with write access to the +zone, forever. In dstack that token lives inside the gateway CVM. Attestation +covers what the CVM is running, but a token is a token: anything that gets hold +of it can rewrite the zone, including records that have nothing to do with +certificates. + +`dns-persist-01` moves the proof out of the issuance loop. The zone owner +publishes one record naming the CA and the ACME account allowed to issue: + +```dns +_validation-persist.example.com. IN TXT "letsencrypt.org; accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1234567890" +``` + +The account key proves who is asking, the record proves the zone owner agreed, +and neither changes between orders. certbot reads DNS and never writes it, so +the CVM holds no DNS credential and the zone can be hosted anywhere — no +Cloudflare account, no API token, no provider integration. + +> **Experimental.** `dns-persist-01` is specified in +> [draft-ietf-acme-dns-persist-01][draft], which is still changing: an open +> working-group issue may add a client-key-derived value to the record, and +> Let's Encrypt has said it will not deploy the challenge to production until +> that is resolved. It is live on Let's Encrypt **staging** and in +> [Pebble][pebble]. Treat the record format as unstable, and expect to +> republish when the draft settles. + +[draft]: https://datatracker.ietf.org/doc/html/draft-ietf-acme-dns-persist-01 +[pebble]: https://github.com/letsencrypt/pebble + +## What the record means + +| Part | Effect | +| --- | --- | +| `letsencrypt.org` | Issuer Domain Name. A CA ignores records naming a different issuer, so one label can hold records for several CAs. | +| `accounturi=` | The ACME account authorized to issue. Compared byte for byte — no case folding, no URI normalization. | +| `policy=wildcard` | Extends the record to `*.example.com`. Without it the CA authorizes `example.com` alone and refuses wildcard orders. | +| `persistUntil=` | Optional UNIX timestamp after which the CA stops accepting the record. | + +A wildcard order authorizes from its base name — `*.example.com` is validated +against `_validation-persist.example.com`, not +`_validation-persist.*.example.com` — so one record covers a name and its +wildcard. + +Two things about the syntax bite in practice, because a CA rejects the whole +record rather than ignoring the offending part: **no trailing semicolon**, and +**no whitespace inside a value**. certbot renders records that satisfy both; +copy them verbatim rather than retyping. + +The scope stops at the names above. Let's Encrypt does not walk up the tree, so +a record on `example.com` does not authorize `sub.example.com` — give each base +name its own record. + +## Standalone certbot + +`certbot` never writes DNS in this mode, so setup is: create the account, read +the records off it, publish them, then issue. + +```toml +# certbot.toml +workdir = "/var/lib/certbot" +acme_url = "https://acme-staging-v02.api.letsencrypt.org/directory" +challenge = "dns-persist-01" +issuer_domain_name = "letsencrypt.org" +# auto_set_caa promises certbot keeps CAA in sync, which it cannot do without +# write access. Leave it off and publish the CAA records below by hand. +auto_set_caa = false +domains = ["example.com", "*.example.com"] +renew_interval = 3600 +renew_days_before = 10 +renew_timeout = 120 +# Capped at half of renew_timeout, so this one waits 60s, not 300s. The whole +# order -- the wait, the authorizations, finalize, and fetching the certificate +# -- has to fit inside renew_timeout, and a wait that outlasts it takes the +# timeout with it, losing the warning naming the record that was missing. +# Raising this alone changes nothing; raise renew_timeout with it. +max_dns_wait = 300 +``` + +`cf_api_token` is unused and can be left out; certbot warns if one is set. + +```console +$ certbot init -c certbot.toml +INFO certbot::bot: creating new ACME account +INFO certbot::bot: created new ACME account: https://acme-staging-v02.api.letsencrypt.org/acme/acct/1234567890 + +$ certbot dns-records -c certbot.toml +_validation-persist.example.com. IN TXT "letsencrypt.org; accounturi=https://acme-staging-v02.api.letsencrypt.org/acme/acct/1234567890; policy=wildcard" +example.com. IN CAA 0 issue "letsencrypt.org;validationmethods=dns-persist-01;accounturi=https://acme-staging-v02.api.letsencrypt.org/acme/acct/1234567890" +example.com. IN CAA 0 issuewild "letsencrypt.org;validationmethods=dns-persist-01;accounturi=https://acme-staging-v02.api.letsencrypt.org/acme/acct/1234567890" +``` + +Publish all three, wait for them to propagate, then issue: + +```console +$ certbot renew --once -c certbot.toml +INFO certbot::acme_client: requesting new certificates for example.com, *.example.com +INFO certbot::bot: created new certificate +``` + +Renewals need nothing further. The record stays, and `certbot renew` reuses it +for every order. + +The CAA records are optional but recommended — they stop any other account, at +Let's Encrypt or elsewhere, from being issued for your name. Note the +`validationmethods=dns-persist-01` in them: a CAA record left pinned to +`dns-01` refuses every `dns-persist-01` order, so switching methods means +updating CAA and the validation record together. + +### When issuance fails + +certbot checks its own resolver before starting an order, and says exactly what +it expected to find: + +``` +WARN certbot::acme_client: no TXT record at _validation-persist.example.com matches the expected value: letsencrypt.org; accounturi=https://acme-staging-v02.api.letsencrypt.org/acme/acct/1234567890; policy=wildcard +Error: order is invalid: API error: Checking DNS-PERSIST-01 challenge TXT record with issuer-domain-name "letsencrypt.org": accounturi mismatch: expected "https://acme-staging-v02.api.letsencrypt.org/acme/acct/1234567890", got "https://acme-staging-v02.api.letsencrypt.org/acme/acct/9876543210" (urn:ietf:params:acme:error:unauthorized) +``` + +The check is advisory and never blocks an order: certbot's resolver is not the +CA's, and its expectation can be stricter than what the CA would accept. A +warning with a successful issuance underneath it is a resolver difference, not +a problem. A record that genuinely does not match costs the whole DNS budget +before the order is sent, because the check waits it out first -- that is +`max_dns_wait`, or half of `renew_timeout` when that is smaller, which for the +configuration above is 60s. certbot logs the budget it settled on at `debug`. + +If the CA rejects the order, compare the published record against +`certbot dns-records` character by character. The usual causes are a stale +`accounturi` after the account was recreated, a missing `policy=wildcard` on a +wildcard order, and an issuer domain name that does not match the CA. + +## dstack-gateway + +A ZT domain picks its method with the `challenge` field, which defaults to +`dns-01` — existing domains are unaffected: + +```json +{ "domain": "app.example.com", "port": 443, "challenge": "dns-persist-01" } +``` + +Such a domain needs no `dns_cred_id`, and the gateway CVM never receives a DNS +credential for it. `GetZtDomain` and `ListZtDomains` return the records to +publish in `required_dns_records`, and the gateway logs them whenever it cannot +write DNS itself: + +``` +WARN cert[app.example.com]: publish this record by hand: _validation-persist.app.example.com. IN TXT "letsencrypt.org; accounturi=...; policy=wildcard" +``` + +The gateway issues for `*.{domain}` only, so each domain needs one validation +record with `policy=wildcard`. + +### Setup order + +The records name the ACME account, so the account comes first: + +1. `SetCertbotConfig` — `acme_url`, and `issuer_domain_name` if the CA is not + Let's Encrypt. +2. `RotateAcmeCredentials` — registers the account and returns its `account_uri`. + It needs no ZT domain: registration proves nothing about a domain. Every call + registers a *new* account, so call it once here; running it again later is a + rotation, with the consequences described below. +3. `AddZtDomain` with `challenge: "dns-persist-01"`. No `dns_cred_id`: the + domain needs none, and the gateway CVM never receives one for it. +4. Read the records — the dashboard shows them when the domain is added and + again under its **Records** button, and `GetZtDomain`, `ListZtDomains` and + the gateway log all carry them — then publish the `_validation-persist` TXT + record, plus the CAA records if you want issuance pinned to this account. +5. `RenewZtDomainCert`. Renewals from here need nothing further. + +Adding another dns-persist-01 domain later starts at step 3: the account already +exists, so the new domain's records are available as soon as it is. A domain's +challenge is chosen when it is added — in the dashboard's ZT-Domain form as well +as over the API — and carried forward by every edit. `UpdateZtDomain` replaces +the whole record, so `challenge` is optional there and omitting it means "leave +it as it is": a caller that predates the field — a cached dashboard bundle, a +script, an older SDK — cannot downgrade a `dns-persist-01` domain to `dns-01` by +editing something else, which its hand-published CAA would then refuse. Send the +field explicitly to switch a domain over. + +### Naming the CA + +For a non-production ACME server, set `issuer_domain_name` in the global certbot +config — `Admin.SetCertbotConfig`, or the field of that name in the dashboard's +Certbot Configuration — to whatever that server puts in `issuer-domain-names`, +`pebble.letsencrypt.org` for Pebble. Empty means `letsencrypt.org`. + +It also names the CA in the CAA records certbot writes for `dns-01` domains, so +one setting covers both challenges rather than pinning CAA to Let's Encrypt +while orders go somewhere else. `acme_url` is one setting for the whole +deployment; this follows it. + +The value is checked when it is set. It ends up verbatim inside an RFC 8659 +`issue-value`, and publishing CAA deletes the records it replaces before writing +the new ones, so a name carrying a space or a `;` would leave the zone holding a +malformed `issue` property with nothing valid behind it — CAA that forbids every +issuer. Anything that is not a DNS name is refused by `SetCertbotConfig`, and by +`certbot` at startup. + +Publishing CAA installs a temporary `;` guard, deletes the records it replaces, +writes the new ones and drops the guard. A run interrupted part-way leaves the +guard behind, which denies every issuer until a later run finishes the job — so +each run now sweeps stale guards before installing its own, and a rerun is +enough to recover. + +### How long the check waits + +The self-check before each order waits for the record to become visible, but +never longer than half of `renew_timeout`, the budget for the whole renewal. +Past that the outer timeout fires first and the renewal ends as `certificate +request timed out` with no mention of the record it could not find — the +opposite of what the check is for. Both values defaulted to 300s in the gateway +(and the CLI wrapped a 300s wait in a 120s budget), so this was the default +behaviour rather than an edge case. + +Two operations behave differently on these domains: + +- **`SetCaa`** skips them. There is nothing to reconcile without write access; + the records are logged instead, and the summary reports how many were left to + the operator. +- **`RotateAcmeCredentials`** is not self-service. Rotation moves the cluster + to a new ACME account, and every `_validation-persist` record still names the + old one, so orders for those domains fail until the operator republishes. The + response returns the new records in `required_dns_records` and the gateway + logs them; publish before the next renewal comes due. + + A rotation that registers the account but cannot re-pin every `dns-01` + domain's CAA still succeeds — the account exists and the cluster is on it — + and names those domains in `repin_failed_domains`. Finish with `SetCaa` + rather than another rotation, which would register yet another account + against a rate-limited quota. The dashboard reports both lists in one dialog + after a rotation, with a **Run SetCaa** button for the second. + +## Related + +- [dstack-gateway](dstack-gateway.md) — gateway architecture and TLS termination +- [deployment.md](deployment.md#4-zero-trust-https-optional) — the `dns-01` setup this replaces diff --git a/docs/confidential-ai.md b/docs/confidential-ai.md index 8d4e2f8a3..f418b805f 100644 --- a/docs/confidential-ai.md +++ b/docs/confidential-ai.md @@ -144,7 +144,7 @@ from dstack_sdk import DstackClient client = DstackClient() info = client.info() -print(f"Compose hash: {info.compose_hash}") +print(f"Compose hash: {info.compose_hash.hex()}") # Data provider compares this against the docker-compose they reviewed ``` diff --git a/docs/deployment.md b/docs/deployment.md index be439e30a..7f774baba 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -42,10 +42,10 @@ If you skip the KMS allowlist step, the VM may boot and the onboard UI may still 1. Set up TDX host with dstack-vmm 2. Deploy KMS as CVM (with auth server, capture its attestation info, and allowlist the KMS `mrAggregated` before bootstrap) 3. Deploy Gateway as CVM +4. [Zero Trust HTTPS](#4-zero-trust-https) - the gateway cannot serve an app over TLS until it holds a certificate for the domain **Optional Add-ons:** -4. [Zero Trust HTTPS](#4-zero-trust-https-optional) 5. [Certificate Transparency monitoring](#5-certificate-transparency-monitoring-optional) 6. [Multi-node deployment](#6-multi-node-deployment-optional) 7. [On-chain governance](./onchain-governance.md) - Smart contract-based authorization @@ -74,6 +74,11 @@ reuse = true image_path = "./images" run_path = "./run/vm" +[auth] +enabled = true +# generate with: openssl rand -hex 32 +tokens = [""] + [cvm] kms_urls = [] gateway_urls = [] @@ -93,6 +98,10 @@ address = "vsock:2" port = 10000 ``` +> **VMM API authentication (required for non-localhost binds).** Since [PR #796](https://github.com/Dstack-TEE/dstack/pull/796), the `[auth]` token guards the *entire* VMM surface — creating and stopping CVMs, the web UI, and all pRPC calls — not just `/logs`. Because the example above binds `tcp:0.0.0.0:9080` (reachable off the host), you MUST enable `[auth]`; otherwise the whole control API is exposed unauthenticated. Auth is fail-closed: with `enabled = true` and no usable credential, requests are rejected. +> +> Clients authenticate with `Authorization: Bearer ` (or the `X-Admin-Token: ` header). Instead of inline `tokens`, you can point to an Apache bcrypt htpasswd file with `htpasswd_file = "/etc/dstack/admin.htpasswd"` (create it with `htpasswd -B -c /etc/dstack/admin.htpasswd admin`). Only bcrypt (`-B`) entries are accepted. + Download guest images from [dstack guest-OS releases](https://github.com/Dstack-TEE/dstack/releases) and extract to `./images/`. > For reproducible builds and verification, see the [Security Model](./security/security-model.md). @@ -113,6 +122,8 @@ Production KMS requires: #### Auth Server Options +> **Note:** the boot-authorization webhook below (auth-simple / auth-eth) is a *different* mechanism from the KMS admin-API authentication. The webhook allowlists which CVMs may boot and receive keys; the admin API (`[core.admin]` in `kms.toml`) guards operator RPCs. See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication). + | Server | Use Case | Configuration | |--------|----------|---------------| | [auth-simple](../dstack/kms/auth-simple/) | Config-file-based whitelisting | JSON config file | @@ -310,33 +321,103 @@ kms_urls = ["https://kms.example.com:9201"] gateway_urls = ["https://gateway.example.com:9202"] ``` +`gateway_urls` is a failover list for one gateway cluster. To register a CVM +with independently operated clusters, configure explicit groups instead. Each +cluster gets a separate WireGuard interface and key pair; their WireGuard +address ranges must not overlap. All clusters must run the gateway app identity +authorized by the CVM's KMS-issued app keys. + +Clusters refresh independently. If one cluster is unavailable, its last +working WireGuard configuration remains active while other clusters continue +to register and update normally. + +`gateway_urls` and `gateway_clusters` are mutually exclusive in the VMM +configuration. The VMM refuses to start if both are non-empty. For compatibility +with sys-config files produced elsewhere, the guest prefers `gateway_clusters` +and logs a warning when both forms are present. + +```toml +[cvm] +kms_urls = ["https://kms.example.com:9201"] + +[[cvm.gateway_clusters]] +name = "primary" +urls = [ + "https://gateway-a.example.com:9202", + "https://gateway-b.example.com:9202", +] +[[cvm.gateway_clusters]] +name = "secondary" +urls = ["https://gateway-c.example.com:9202"] +``` + Restart dstack-vmm to apply changes. --- -### 4. Zero Trust HTTPS (Optional) +### 4. Zero Trust HTTPS -Generate TLS certificates inside the TEE with automatic CAA record management. +The gateway issues its own certificates from inside the CVM. `dstack-gateway` +links the `certbot` crate directly: it answers dns-01 challenges with the +Cloudflare credential you give it, and keeps the ACME account key and every +certificate in the CVM's WaveKV store. Neither key is ever written to the host, +which is what makes the monitoring in step 5 worth running — every certificate +the CT logs show for your domain should carry a public key the gateway +published. Nothing on the host issues or holds these certificates; the `certbot` +CLI under `dstack/certbot/cli` is a testing tool for the same crate and has no +part in this path. -Configure in `build-config.sh`: +`bootstrap-cluster.sh` configures all of it, reading `CF_API_TOKEN`, +`SRV_DOMAIN` and `ACME_STAGING` from the `.env` you filled in during step 3 and +calling the gateway's admin API: ```bash -GATEWAY_CERT=${CERTBOT_WORKDIR}/live/cert.pem -GATEWAY_KEY=${CERTBOT_WORKDIR}/live/key.pem -CF_API_TOKEN= -ACME_URL=https://acme-v02.api.letsencrypt.org/directory +cd dstack/gateway/dstack-app/ +bash bootstrap-cluster.sh ``` -Run certbot: +| RPC | What it sets | +|---|---| +| `SetCertbotConfig` | ACME directory URL and the renewal schedule | +| `CreateDnsCredential` | the Cloudflare token used for dns-01, as the default credential | +| `AddZtDomain` | a domain to keep a wildcard certificate for | + +Run it once per cluster. Additional nodes receive all three through cluster +sync, so do not repeat it per node. + +Every name the gateway terminates TLS on needs its own ZT domain. The script +adds `SRV_DOMAIN`; if app URLs sit one level deeper — `-.gateway.example.com` +— then `gateway.example.com` needs an entry of its own, because a wildcard +certificate for `*.example.com` does not cover subdomains of subdomains. + +Certificates are requested on the next renewal round rather than the moment a +domain is added. Watch for them to arrive: ```bash -RUST_LOG=info,certbot=debug ./certbot renew -c certbot.toml +ADMIN_ADDR=127.0.0.1:9203 # GATEWAY_ADMIN_RPC_ADDR in .env +curl -sf -H "Authorization: Bearer $ADMIN_API_TOKEN" \ + "http://$ADMIN_ADDR/prpc/ListZtDomains" | jq '.domains[] | {domain: .config.domain, cert: .cert_status}' ``` -This will: -- Create an ACME account -- Set CAA DNS records on Cloudflare -- Request and auto-renew certificates +`has_cert: true` with a `not_after` roughly 90 days out means the domain is +served. `POST /prpc/RenewCert` forces a round immediately instead of waiting +for `renew_interval_secs`. + +Pin issuance to your own ACME account with `POST /prpc/SetCaa`, which writes +CAA records naming Let's Encrypt and the gateway's account URI for every +configured domain. Any other account is then refused by the CA rather than +merely noticed after the fact by step 5. + +Start on Let's Encrypt staging (`ACME_STAGING=yes`), whose certificates are not +browser-trusted but whose rate limits leave room for mistakes. Switching to +production takes two calls: `SetCertbotConfig` with the production directory +URL, then `RotateAcmeCredentials` to register an account there and re-pin every +domain's CAA to it. Renewals refuse to run while the stored account and the +configured ACME URL disagree, so do not skip the rotation. + +For the same flow driven by hand, one curl at a time, see the +[Gateway service setup tutorial](./tutorials/gateway-service-setup.md#step-4-bootstrap-admin-api). +For a multi-node cluster, see [Cluster deployment](../dstack/gateway/docs/cluster-deployment.md). --- @@ -428,6 +509,8 @@ curl http://:9203/finish > > If you skip this, `Onboard.Onboard` or later trusted RPCs will fail with KMS authorization errors. +> **Admin authentication.** Onboarding itself is gated by attestation and the authorization backend above — not by a token. Separately, the KMS *admin* RPCs (for example `ClearImageCache`) are served on a dedicated `[core.admin]` listener behind the shared HTTP authenticator, just like the VMM and gateway: set `[core.admin] enabled = true` with an `auth_token` (or the `DSTACK_KMS_ADMIN_TOKEN` / `ADMIN_API_TOKEN` env vars), and clients send `Authorization: Bearer ` or `X-Admin-Token`. Enabled with neither `auth_token` nor `htpasswd_file` (and `insecure_no_auth = false`) fails closed — the KMS refuses to start. See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication). + --- ## Deploying Apps diff --git a/docs/design-and-hardening-decisions.md b/docs/design-and-hardening-decisions.md index af5dd0ae6..b6ffbb9cd 100644 --- a/docs/design-and-hardening-decisions.md +++ b/docs/design-and-hardening-decisions.md @@ -1,5 +1,11 @@ # Design and Hardening Decisions in the dstack Yocto Layer +> [!NOTE] +> This document describes the deprecated Yocto backend. The default guest-OS +> backend is mkosi (`os/mkosi/`); see +> [`../os/mkosi/README.md`](../os/mkosi/README.md) for how its image is built +> and where it differs. + ## Overview The dstack-owned Yocto layer under `os/yocto/layers/meta-dstack/` is designed to create a minimally secure image for booting Confidential Virtual Machines (CVMs). Our design philosophy prioritizes attack surface reduction while maintaining TDX-aware functionality. This document outlines the architectural decisions and trade-offs made during development. diff --git a/docs/development-without-tee.md b/docs/development-without-tee.md new file mode 100644 index 000000000..560f99722 --- /dev/null +++ b/docs/development-without-tee.md @@ -0,0 +1,233 @@ +# Develop with dstack without TEE hardware + +Development guest images can run dstack's normal guest setup on a KVM machine +that has no TDX or SEV-SNP support. The VMM starts the guest with `no_tee`, the +guest supplies the TDX ABI through `dstack-tee-simulator`, and `swtpm` provides +persistent TPM-backed application keys. This is suitable for development and +integration testing, not for production workloads or secrets. + +## What this mode tests + +The development simulator provides configfs-tsm reports, RTMR extension files, +and a CCEL event log. Guest preparation, measurements, application key setup, +encrypted persistent storage, the guest agent, and Docker Compose therefore +use their normal code paths. + +It does not provide hardware isolation or a valid hardware-signed quote. The +host controls QEMU and swtpm, and a production verifier or KMS must reject the +simulated quote. + +## Build the development image on the host + +TEE hardware is not needed for the build. Follow the prerequisites in +[Build the dstack guest OS](building-guest-os.md), then run this from the +repository root: + +```bash +FLAVORS="dev" ./os/mkosi/repro-build/repro-build.sh +``` + +The artifact used below is +`os/mkosi/repro-build/build/out/dev/dstack-dev-.tar.gz`. Confirm that +it really is a development image: + +```bash +mkdir -p ~/.dstack-vmm/image +tar -xzf os/mkosi/repro-build/build/out/dev/dstack-dev-*.tar.gz \ + -C ~/.dstack-vmm/image +jq '{version, git_revision, is_dev}' \ + ~/.dstack-vmm/image/dstack-dev-*/metadata.json +``` + +Expected output includes `"is_dev": true`. Use a development image for this +workflow because production images do not include `dstack-tee-simulator`. + +## Install and configure the VMM + +Install QEMU and swtpm on the development host. On Ubuntu: + +```bash +sudo apt-get update +sudo apt-get install -y qemu-system-x86 swtpm swtpm-tools jq +test -r /dev/kvm && test -w /dev/kvm +``` + +Add the current user to the `kvm` group and log in again if the `/dev/kvm` +check fails because of its permissions. + +Build the VMM and supervisor from the same checkout as the image: + +```bash +cargo build --manifest-path dstack/Cargo.toml --release \ + -p dstack-vmm -p supervisor +``` + +Install both binaries and the CLI on the machine that will run QEMU. You can +also run them directly from the checkout during development: + +```bash +mkdir -p ~/.dstack-vmm +install -Dm755 dstack/target/release/dstack-vmm ~/.local/bin/dstack-vmm +install -Dm755 dstack/target/release/supervisor ~/.local/bin/supervisor +install -Dm755 dstack/vmm/src/vmm-cli.py ~/.local/bin/dstack +mkdir -p ~/.dstack-vmm +cp dstack/vmm/vmm.toml ~/.dstack-vmm/vmm.toml +``` + +Set these values in `~/.dstack-vmm/vmm.toml`: + +```toml +[image] +path = "/home/USER/.dstack-vmm/image" + +[cvm] +qemu_path = "/usr/bin/qemu-system-x86_64" + +[cvm.networking] +mode = "user" + +[cvm.tee_simulator] +# Development credential only. Use a different random 32-byte hex seed for +# each isolated test environment; never use it for production secrets. +mock_attestation_seed = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +collateral_base_url = "http://10.0.2.2:18088" + +[supervisor] +exe = "/home/USER/.local/bin/supervisor" +``` + +These node-local settings provide development credentials and collateral URLs; +they do not enable simulation globally. Each deployment selects its simulated +platform with `--simulated-tee`. The VMM then writes an instance-specific +`.tee-simulator.json`, and the guest starts the simulator only when that file +is present in the host share. + +Start the VMM from a stable working directory because its default API socket is +relative to that directory: + +```bash +mkdir -p ~/.dstack-vmm/run +cd ~/.dstack-vmm/run +dstack-vmm -c ../vmm.toml +``` + +In another terminal, check that the development image is visible: + +```bash +cd ~/.dstack-vmm/run +dstack lsimage --json | jq '.[] | {name, version, is_dev}' +``` + +## Launch with no TEE and swtpm + +Create a small stateful workload: + +```yaml +# docker-compose.yml +services: + state-test: + image: alpine:3.20 + command: + - sh + - -c + - | + date -Iseconds >> /data/boots + touch /data/persistent-marker + sleep infinity + volumes: + - state-data:/data +volumes: + state-data: {} +``` + +Convert it to app-compose JSON and select the TPM key provider: + +```bash +dstack compose \ + --name swtpm-persistence \ + --docker-compose docker-compose.yml \ + --key-provider tpm \ + --public-logs \ + --public-sysinfo \ + --output app-compose.json +``` + +Deploy with the development image, no KMS, and an instance-specific simulated +TEE platform: + +```bash +dstack deploy \ + --name swtpm-persistence \ + --image dstack-dev- \ + --compose app-compose.json \ + --vcpu 2 --memory 3G --disk 10G \ + --simulated-tee dstack-tdx +``` + +Successful output contains a VM ID. `dstack info VM_ID` should eventually show +`Boot Progress: done`. The QEMU command line should contain the swtpm frontend +and no TDX guest object: + +```bash +ps aux | grep '[q]emu-system' | grep -E -- '-tpmdev|tpm-tis' +ps aux | grep '[s]wtpm socket' +``` + +The VMM stores both durable components below the VM work directory: + +```text +~/.dstack-vmm/vm/VM_ID/hda.img +~/.dstack-vmm/vm/VM_ID/swtpm/tpm2-00.permall +``` + +Do not delete either path if the VM must restart with the same state. + +## Verify encrypted storage and restart persistence + +First check the guest preparation log: + +```bash +dstack logs VM_ID -n 300 | grep -E \ + 'Generating app keys from TPM|Filesystem options|LUKS2 header|Device mapper' +``` + +A successful first boot includes output like: + +```text +Generating app keys from TPM +Filesystem options: encryption=true, filesystem=Zfs +Mounting tmpfs for in-memory LUKS header +Loading the LUKS2 header +Device mapper /dev/mapper/dstack_data_disk is ready +``` + +Record the instance ID, then perform a graceful restart: + +```bash +dstack info VM_ID | grep 'Instance ID' +dstack stop VM_ID +dstack start VM_ID +``` + +Wait for `Boot Progress: done`, then inspect the recent events and boot log: + +```bash +dstack info VM_ID +dstack logs VM_ID -n 300 | grep -E \ + 'mounting data disk|Loading the LUKS2 header|Device mapper|Container .* Started' +``` + +The restart passes when all of the following hold: + +1. the instance ID is unchanged; +2. the event stream says `mounting data disk`, not `initializing data disk`; +3. the LUKS mapping becomes ready without reformatting the disk; +4. Docker starts the existing container and volume, and + `/data/persistent-marker` and the earlier entries in `/data/boots` remain. + +## Common failures + +`tpm key provider requested but swtpm is not installed` means `swtpm` is not on +the VMM user's `PATH`. If QEMU reports that KVM is unavailable, confirm that +hardware virtualization is enabled and that `/dev/kvm` is writable by the VMM +user. diff --git a/docs/dstack-gateway.md b/docs/dstack-gateway.md index 78ad911bf..fe4992b29 100644 --- a/docs/dstack-gateway.md +++ b/docs/dstack-gateway.md @@ -2,41 +2,125 @@ > **This guide is for self-hosted deployments** on your own TDX hardware. For cloud deployments, see [Quickstart](./quickstart.md). -To set up dstack-gateway for production, you need a wildcard domain and SSL certificate. +To set up dstack-gateway for production, you need a wildcard domain and a +Cloudflare API token. You do not need to obtain a certificate yourself: the +gateway links the `certbot` crate and runs ACME over dns-01 in its own process, +keeping the ACME account key and every certificate in its WaveKV store. The +`certbot` CLI under `dstack/certbot/cli` is a testing tool for the same crate +and has no part in this path. -## Step 1: Setup wildcard domain +## Step 1: Set up the wildcard domain Set up a second-level wildcard domain using Cloudflare; make sure to disable proxy mode and use **DNS Only**. ![add-wildcard-domain](./assets/tproxy-add-wildcard-domain.jpg) -## Step 2: Request a Wildcard Domain SSL Certificate with Certbot +Then create an API token that can edit this zone's DNS records. The gateway uses +it to publish the `_acme-challenge` TXT records that answer dns-01, and the CAA +records that pin issuance to its own ACME account. -You need to get a Cloudflare API Key and ensure the API can manage this domain. +## Step 2: Configure `gateway.toml` -Open your `certbot.toml`, and update these fields: +Focus on these fields in the `core.proxy` section: -- `acme_url`: change to `https://acme-v02.api.letsencrypt.org/directory` -- `cf_api_token`: Obtain from Cloudflare +- `base_domain`: the wildcard domain for the proxy +- `listen_addr` & `listen_port`: listen on `0.0.0.0` and preferably `443` in production. If using another port, specify it in the URL (see [URL Format](#url-format)) -## Step 3: Run Certbot Manually and Get First SSL Certificates +For example, if your base domain is `gateway.example.com`, app ID is ``, listening on `80`, and dstack-gateway is on port 7777, the URL would be `https://-80.gateway.example.com:7777` + +Leave `cert_chain` and `cert_key` unset. They load a certificate you already +have from disk at startup, for the case where something else issues it; the +gateway's own issuance does not use them and does not write them. + +Two more sections matter for certificates: -```shell -./certbot set-caa -./certbot renew +```toml +[core.admin] +enabled = true +address = "127.0.0.1:9016" +auth_token = "" + +[core.sync] +data_dir = "/var/lib/dstack-gateway/data" ``` -## Step 4: Update `gateway.toml` +The admin API is where the ACME settings, the Cloudflare token and the domain +list live — they are stored in the gateway's KV store and there is no file to +put them in, so certificates cannot be issued without it. See +[Admin API authentication](#admin-api-authentication) for the credential +options. `data_dir` is where the ACME account key and the issued certificates +are persisted; point it somewhere writable that survives restarts, or the +gateway asks the CA for a fresh certificate every time it starts and will run +into Let's Encrypt's rate limits. The section is named for cluster sync, but +this store is used whether or not `enabled` is set. + +Start the gateway. + +## Step 3: Give the gateway its ACME configuration + +Open the admin dashboard at `http://` and fill in **Certbot +Configuration**, **DNS Credentials** and **ZT-Domains**, or do the same over the +admin API: + +```bash +ADMIN_ADDR=127.0.0.1:9016 +AUTH=(-H "Authorization: Bearer $ADMIN_API_TOKEN") + +# Start on staging: its certificates are not browser-trusted, but its rate +# limits leave room for mistakes. +curl -sf -X POST "${AUTH[@]}" "http://$ADMIN_ADDR/prpc/SetCertbotConfig" \ + -H "Content-Type: application/json" \ + -d '{"acme_url":"https://acme-staging-v02.api.letsencrypt.org/directory", + "renew_interval_secs":3600,"renew_before_expiration_secs":864000, + "renew_timeout_secs":300}' + +curl -sf -X POST "${AUTH[@]}" "http://$ADMIN_ADDR/prpc/CreateDnsCredential" \ + -H "Content-Type: application/json" \ + -d '{"name":"cloudflare","provider_type":"cloudflare", + "cf_api_token":"'"$CF_API_TOKEN"'","set_as_default":true}' + +curl -sf -X POST "${AUTH[@]}" "http://$ADMIN_ADDR/prpc/AddZtDomain" \ + -H "Content-Type: application/json" \ + -d '{"domain":"gateway.example.com","port":443,"priority":100}' +``` -Focus on these five fields in the `core.proxy` section: +Add a ZT domain for every name the gateway terminates TLS on. An entry for +`gateway.example.com` gets a certificate covering `*.gateway.example.com`, which +is what app URLs live under — a certificate for `*.example.com` would not, since +a wildcard does not span a further label. `port` is the port that domain is +served on and `priority` breaks ties when more than one entry could be the +default base domain. -- `cert_chain` & `cert_key`: Point to the certificate paths from the previous step -- `base_domain`: The wildcard domain for proxy -- `listen_addr` & `listen_port`: Listen to `0.0.0.0` and preferably `443` in production. If using another port, specify it in the URL +Certificates are requested on the next renewal round rather than the moment a +domain is added. Watch for them to arrive: -For example, if your base domain is `gateway.example.com`, app ID is ``, listening on `80`, and dstack-gateway is on port 7777, the URL would be `https://-80.gateway.example.com:7777` +```bash +curl -sf "${AUTH[@]}" "http://$ADMIN_ADDR/prpc/ListZtDomains" \ + | jq '.domains[] | {domain: .config.domain, cert: .cert_status}' +``` + +`has_cert: true` with a `not_after` roughly 90 days out means the domain is +served. `POST /prpc/RenewCert` forces a round immediately instead of waiting for +`renew_interval_secs`. + +Pin issuance with `POST /prpc/SetCaa`, which writes CAA records naming Let's +Encrypt and the gateway's ACME account URI for every configured domain, so no +other account can have a certificate issued for them. -### URL Format +Once the gateway serves traffic on staging certificates, switch to production: +`SetCertbotConfig` with `https://acme-v02.api.letsencrypt.org/directory`, then +`RotateAcmeCredentials` to register an account there and re-pin every domain's +CAA to it. Renewals refuse to run while the stored account and the configured +ACME URL disagree, so do not skip the rotation. + +## Step 4: Adjust Configuration in `vmm.toml` + +Open `vmm.toml` and adjust dstack-gateway configuration in the `gateway` section: + +- `base_domain`: Same as `base_domain` from `gateway.toml`'s `core.proxy` section +- `port`: Same as `listen_port` from `gateway.toml`'s `core.proxy` section + +## URL Format The gateway supports the following URL format: - `[-][].` @@ -57,9 +141,72 @@ Examples: Note: The `s` and `g` suffixes cannot be used together -## Step 5: Adjust Configuration in `vmm.toml` +## Admin API authentication -Open `vmm.toml` and adjust dstack-gateway configuration in the `gateway` section: +The gateway exposes a separate admin API (used for sync, WireGuard peer management, and other operator RPCs). Configure it in the `core.admin` section of `gateway.toml`: -- `base_domain`: Same as `base_domain` from `gateway.toml`'s `core.proxy` section -- `port`: Same as `listen_port` from `gateway.toml`'s `core.proxy` section +```toml +[core.admin] +enabled = true +address = "0.0.0.0:9016" +# generate with: openssl rand -hex 32 +auth_token = "" +# alternatively, an Apache bcrypt htpasswd file (htpasswd -B -c admin.htpasswd admin) +# htpasswd_file = "/etc/dstack/gateway-admin.htpasswd" +insecure_no_auth = false +``` + +- `enabled`: enable the admin API server. +- `address`: bind address/port for the admin API. +- `auth_token`: shared admin token. It can also be supplied via the environment variables `DSTACK_GATEWAY_ADMIN_TOKEN` or `ADMIN_API_TOKEN` instead of the config file. The older name `admin_token` is still accepted. +- `htpasswd_file`: path to an Apache bcrypt htpasswd file (create with `htpasswd -B -c admin.htpasswd admin`); only bcrypt entries are accepted. Can be used instead of, or alongside, `auth_token`. +- `insecure_no_auth`: development-only escape hatch that disables admin authentication. Never enable it on a network-reachable admin interface. + +The admin server is fail-closed: if it is enabled with no `auth_token` and no `htpasswd_file`, and `insecure_no_auth` is `false`, it refuses to start rather than exposing an unauthenticated admin API. + +Clients authenticate by sending `Authorization: Bearer ` or the `X-Admin-Token: ` header. + +## Metrics + +The admin server exposes Prometheus metrics at `GET /metrics`. It is part of the +admin API, so it is only reachable when `core.admin.enabled` is true and it +requires the same credentials — unless `insecure_no_auth` is set, which exposes +it along with the rest of the admin API. The series name domains, node ids and +instance counts, which is topology that should not be readable without +authentication. + +```yaml +scrape_configs: + - job_name: dstack-gateway + static_configs: + - targets: [""] + authorization: + credentials: "" +``` + +### Cluster-scoped vs node-local series + +`dstack_gateway_cluster_*` describes replicated state: every node in the cluster +reports the same value, so summing across targets multiplies it by the number of +nodes. Everything else describes what one process did and sums normally. + +```promql +# Instances in the routing table — replicated, so take one node's view +max(dstack_gateway_cluster_instances) + +# Connections across the fleet — node-local, so add them up +sum(dstack_gateway_connections) + +# Nodes disagreeing about who is up: this is the replication-lag signal +max(dstack_gateway_cluster_nodes_active) - min(dstack_gateway_cluster_nodes_active) +``` + +### Series worth alerting on + +| Metric | Why | +|---|---| +| `dstack_gateway_wg_reconfigure_failures_total` | The gateway could not push a WireGuard config: it failed to render, failed to write, or `wg syncconf` rejected the whole file over one bad peer stanza. Routing updates have stopped reaching the data plane while the gateway still looks healthy. | +| `dstack_gateway_kv_decode_failures_total` | A replicated record that fails to decode is skipped, which makes the CVM behind it silently unroutable. Labelled by key prefix. Alert on `> 0`; the magnitude counts how often a bad record was *read*, not how many are bad, so do not read it as a severity. | +| `dstack_gateway_kv_peer_buffered_logs` | Entries still buffered for a peer. Sustained growth means that peer stopped acknowledging and the two nodes are drifting apart. | +| `dstack_gateway_cluster_cert_not_after_seconds` | Certificate expiry per domain; alert on `- time()` falling under the renewal window. Capped at 256 series — compare `dstack_gateway_cluster_cert_domains` to see whether the cap was hit. | +| `dstack_gateway_kv_persist_failures_total` | Periodic snapshots are failing, so a restart replays a growing WAL. | diff --git a/docs/guest-api-v0.md b/docs/guest-api-v0.md new file mode 100644 index 000000000..50861e12b --- /dev/null +++ b/docs/guest-api-v0.md @@ -0,0 +1,711 @@ +# dstack Guest Agent API v0 (frozen) + +This is the normative specification of the unversioned guest agent API, the +surface dstack 0.6.0 froze at exactly what v0.5.11 served. It defines the URL +scheme, the key derivation function, the signature chain encoding, and the +per-algorithm signing modes, so that a client written against this surface has a +written contract rather than an implementation to read. + +Read it as the contract. The proto file +(`dstack/guest-agent/rpc/proto/agent_rpc.proto`) describes the message shapes; +this document pins the bytes. Where an implementation and this document disagree, +one of them is a bug. + +**New code should target v1.** [`docs/guest-api-v1.md`](./guest-api-v1.md) is the +current API and the one under active development. This document exists because +the frozen surface is permanently closed and every 0.5.x client deployed today +speaks it, not because it is where new work belongs. + +## Scope and status + +The surface is closed. It gains no methods, no fields, no renumbering, and no +semantic changes to what is already there. It exists so a v0.5.x client keeps +working against a 0.6 agent unchanged. Every new capability goes to +`dstack.guest.v1`. + +Three services are frozen: `DstackGuest`, `Worker` and `Tappd`, all three in +the `dstack_guest` proto package. `Tappd` predates the naming scheme the other +two follow. + +The freeze is mechanical, not a convention. `the_frozen_services_match_their_pinned_shape` +in `dstack/guest-agent/rpc/tests/frozen_surface.rs` hashes each frozen service's +descriptor against a pinned SHA-256. The descriptor covers the method list and +the full field list of every message reachable from it, `reserved` ranges +included. +Any addition changes the digest and fails the test, including a wire-compatible +one. That is deliberate: "frozen except for additions" is how `GpuInfo`, +`AttestGpu` and `Worker.Health` accumulated on this surface between v0.5.11 and +0.6.0 without ever being released. +`the_frozen_services_expose_the_v0_5_11_methods` in the same file spells the +method lists out so a failure is readable. + +The descriptor digest pins the shape, not the behaviour. Behaviour is held by +the agent's own tests: `get_key_pins_the_frozen_chain_link` in +`dstack/guest-agent/src/rpc_service.rs` pins a chain link byte for byte, and +`the_internal_v0_mount_is_an_alias_for_the_unversioned_path`, +`the_external_v0_mount_is_an_alias_for_the_unversioned_path` and +`each_internal_mount_serves_only_its_own_surface` in +`dstack/guest-agent/src/server.rs` exercise the real mount table rather than a +restatement of it. + +Two behaviour changes in 0.6.0 are sanctioned and do not alter the wire shape: +`GetQuote` fails on a platform without Intel TDX instead of returning an empty +quote, and `GetTlsKey` rejects a `not_before` that is not earlier than +`not_after`. A third method, `EmitEvent`, always fails. Each is described in +[The remaining methods](#the-remaining-methods). + +## Transport and mounts + +| Listener | Service | Mount | Example path | +|---|---|---|---| +| Internal socket | `DstackGuest` | `/v0` | `/v0/GetKey` | +| Internal socket | `DstackGuest` | `/` | `/GetKey` | +| External | `Worker` | `/prpc/v0` | `/prpc/v0/Info` | +| External | `Worker` | `/prpc` | `/prpc/Worker.Info` | +| Tappd socket | `Tappd` | `/prpc/` | `/prpc/Tappd.Info` | + +The unversioned mounts are aliases, kept so a pre-0.6 client keeps working +unchanged. They are additional mounts of the *same* handler, not a parallel +implementation, so they cannot drift from the versioned path. New code that must +stay on this surface should say `/v0` and `/prpc/v0` explicitly. + +Both `Worker` mounts strip a `Worker.` service-name prefix from the method +segment, so `/prpc/Worker.Info` and `/prpc/Info` both reach `Info`. `Tappd` +strips `Tappd.` the same way. The internal `DstackGuest` mounts do not strip +anything: the method segment is the bare method name. + +The internal socket is `/var/run/dstack.sock`, reachable only by the application +itself, and it is the only place key material is served. The external listener is +reachable by anyone who can route to the CVM, and `Worker` never returns key +material. `Tappd` has its own socket, `/var/run/tappd.sock`. + +Version selection is by URL path and nothing else. There is no header +negotiation and no default-version redirect. + +Both listeners run the same prpc transport. A `POST` carrying +`Content-Type: application/json` takes a JSON body and returns JSON; any other +content type takes a protobuf-encoded request message and returns a +protobuf-encoded response. A `GET` takes its fields as query parameters and +returns JSON. + +In the JSON encoding, a proto `bytes` field is a lowercase hex string. On input +an optional `0x` prefix is accepted and stripped. This applies to every `bytes` +field on this surface, including `GetKeyResponse.key`, each element of a +`signature_chain`, `SignRequest.data`, and `report_data`. + +### Status codes + +These are transport-level and identical on both surfaces. + +| Status | Body | Meaning | +|---|---|---| +| 200 | the response message | Success | +| 404 | the server's own 404 page | No such mount: this agent has no surface at that path | +| 404 | `{"error": "Service not found: "}` | The surface is mounted; it has no such method | +| 400 | `{"error": ""}` | The method ran and failed | +| 413 | `{"error": ""}` | The request body exceeded the configured limit | +| other | `{"error": ""}` | A handler chose the status; the message says why | + +A handler failure is a 400 with the error text in the body. In the protobuf +encoding the same error arrives as a `ProtoError` message rather than JSON. + +## GetKey + +`GetKey` derives an application key from the application root key and returns it +with a two-link signature chain proving where it came from. It is the reason this +document exists: it is the one method on this surface whose output other parties +depend on, and until now its bytes lived only in the implementation. + +`GetKeyArgs` has three fields. + +| Field | Type | Reaches the KDF | +|---|---|---| +| `path` | string | **yes**, verbatim | +| `purpose` | string | no | +| `algorithm` | string | no | + +Only `path` reaches the KDF. `purpose` is echoed into the chain claim of link 0 +and has no other effect: two calls differing only in `purpose` return the same +32 bytes in `key` and differ only in `signature_chain[0]`. `algorithm` selects +how those 32 bytes are *interpreted* and how the public key is encoded; it does +not domain-separate the derivation. + +Derivation is flat. `path` is an opaque byte string used as HKDF `info` and +nothing more. `a/b` is not a child of `a`, no key derived here derives another, +and there is no BIP-32-style hierarchy. The empty string is a valid `path`. + +### The KDF + +```text +salt = "RATLS" (5 bytes, ASCII: 52 41 54 4c 53) +IKM = app root secp256k1 private key (32 bytes, `k256_key` from .appkeys.json) +info = path (the caller's bytes, verbatim) +L = 32 + +key = HKDF-SHA256(salt, IKM, info, L) (RFC 5869: extract, then expand) +``` + +`ra_tls::kdf::derive_key` is the implementation. It takes `context_data` as a +slice of byte slices and hands that slice straight to ring's HKDF `expand`, which +concatenates the parts to form `info`. `GetKey` passes exactly one part, +`request.path.as_bytes()`, so `info` is the raw `path` bytes with no separator, +no tag, and no length prefix. + +The salt is the constant `LEGACY_SALT` in `dstack/ra-tls/src/kdf.rs`. It is the +five ASCII bytes `RATLS`. It is baked into every key deployed before the +versioned API and must never change. + +### Algorithm selection + +`normalize_algorithm` maps the caller's string to a canonical name: `k256` +becomes `secp256k1`, and every other value passes through unchanged. `GetKey` +then matches on the result. + +| `algorithm` sent | Behaviour | +|---|---| +| `secp256k1` | secp256k1 | +| `k256` | alias; identical to `secp256k1`, same key bytes | +| `""` (empty or field absent) | secp256k1, the default | +| `ed25519` | ed25519 | +| `secp256k1_prehashed` | **error.** Prehashing is a signing mode, not a key type; `Sign` accepts it, `GetKey` does not | +| anything else | error, `Unsupported algorithm` | + +An unrecognised value is an error rather than a silent fallback to the default. +Only the literal empty string defaults. + +### Output encoding + +`key` is the 32 derived bytes, raw. It is not a PEM, not a DER, and not +algorithm-tagged; the caller knows which algorithm it asked for. In the JSON +encoding it is a 64-character hex string. + +- **secp256k1**: the 32 bytes are the big-endian private scalar. If it is zero or + at least the group order the call fails with `Failed to parse k256 key`. The + public key is the SEC1 **compressed** point, 33 bytes, `0x02`/`0x03` prefix. +- **ed25519**: the 32 bytes are the RFC 8032 seed, from which the key expands as + usual. The public key is the RFC 8032 raw public key, 32 bytes. + +`GetKeyResponse` has no `public_key` field. A caller derives the public key from +`key` itself, and a verifier of the chain must reproduce the same encoding, as +described below. (v1 added `public_key` to the response for exactly this reason.) + +### One secret, three curves + +The derivation ignores `algorithm`, so `GetKey(path, *, "secp256k1")` and +`GetKey(path, *, "ed25519")` return **the same 32 bytes**. One secret is served +in two representations: a secp256k1 scalar and an ed25519 seed. + +`Tappd.DeriveKey` makes it three. It runs the identical construction -- +HKDF-SHA256, salt `RATLS`, `info` = `path` -- and reads the result as a **P-256** +scalar, which it hands back as a PKCS#8 PEM on `/var/run/tappd.sock`. So +`GetKey("x")` and `Tappd.DeriveKey("x")` are the same secret on a third curve, +and the PEM is a more convenient form to walk off with than either. + +This is a property callers must account for, not a bug in an individual call. +Anyone who can reach either socket can request any of the three interpretations, +so the keys are not independent: compromise of one is compromise of all, and a +protocol that assumes a per-curve key does not get one here. Encoding the +algorithm into `path` -- `backup-signing/secp256k1` and `backup-signing/ed25519` +-- separates the two `GetKey` curves from each other, but not from +`Tappd.DeriveKey`, which will derive the very same secret from the very same +string. The only real boundary is the socket: nothing inside this surface makes +one derived key unreachable through another method. + +This is one of the things v1 changed: the v1 KDF binds the canonical algorithm +name and a version tag into `info` under its own salt, so the two curves never +share a secret. See +[Key derivation](./guest-api-v1.md#key-derivation) and +[Migration from the unversioned API](./guest-api-v1.md#migration-from-the-unversioned-api). + +### The signature chain + +`GetKeyResponse.signature_chain` has exactly two elements. + +```text +[0] app root key signs the v0 key claim (specified here) +[1] KMS root key signs the app root public key +``` + +Both are 65 bytes in the same envelope: + +```text +link = r || s || v (65 bytes) +``` + +`r` and `s` are 32-byte big-endian integers, low-S normalised, and `v` is the +one-byte recovery id in `0..=3`. The recovery byte lets a relying party recover +the signing public key from the link alone. `ra_tls::api_v1::sign_recoverable_keccak256` +produces both links; only the preimage differs. + +**Link 0, the key claim.** + +```text +claim = purpose || ":" || lowercase_hex(public_key) +digest = keccak256(claim) +link0 = r || s || v +``` + +`purpose` is the caller's string verbatim. `public_key` is the derived public key +in the encoding from [Output encoding](#output-encoding), SEC1 compressed for +secp256k1 and RFC 8032 raw for ed25519. It is hex-encoded in lowercase and +appended as **text**, not as raw bytes. The claim is a UTF-8 byte string, since +`purpose` is a proto3 `string`, and the digest is keccak256 over those bytes. + +The claim is a `:`-joined string over a caller-chosen `purpose`, which means an +application can steer much of what its own app root key signs. That is inside the +application's own trust domain, because the caller already holds the derived +key, but it is why v1's claim is length-prefixed instead, and why a v0 claim can +never be mistaken for a v1 one. See +[Why this cannot be forged through v0](./guest-api-v1.md#why-this-cannot-be-forged-through-v0). + +**Link 1, the KMS attestation.** Produced by the KMS, outside the agent, and +passed through byte-for-byte from `k256_signature` in `.appkeys.json`. It is the +same bytes v1 serves in its own `signature_chain[1]`. + +```text +message = "dstack-kms-issued" || ":" || app_id || sec1_compressed(app_root_pubkey) +digest = keccak256(message) +link1 = r || s || v +``` + +`app_id` is the raw app id bytes and the app root public key is SEC1 compressed, +33 bytes. `sign_message` in `dstack/kms/src/crypto.rs` builds the preimage as +`[prefix, b":", appid, message].concat()`. Note there is no separator between +`app_id` and the public key. + +### Verifying a chain + +A relying party holds `key` (or the public key derived from it), a +`signature_chain`, the `(path, purpose, algorithm)` the key was requested under, +and the `app_id`. + +1. **Anchor.** Obtain the KMS root public key from a source you trust + independently of the agent being checked: the `DstackKms` contract's + `kmsInfo().k256Pubkey`, or a value pinned out of band. This step carries the + security of everything below it. An attacker who can answer your query for the + anchor can also mint a self-consistent chain, so reading the anchor from the + KMS you are checking proves nothing. + +2. **Rebuild the claim.** Encode the public key as + [Output encoding](#output-encoding) specifies, hex-encode it in lowercase, and + compute `digest0 = keccak256(purpose || ":" || hex)`. + +3. **Recover the app root key.** Split `signature_chain[0]` into `r`, `s`, `v` + and recover the secp256k1 public key from `(digest0, r, s, v)`. Reject a + non-canonical high-S `s`. Call the result `app_root_pubkey`, SEC1 compressed. + +4. **Rebuild the KMS message.** Compute + `digest1 = keccak256("dstack-kms-issued" || ":" || app_id || app_root_pubkey)`. + +5. **Check link 1.** Verify `signature_chain[1]` over `digest1` against the anchor + from step 1, either by recovering and comparing to the anchor or by verifying + `(r, s)` against it directly. Reject high-S here too. + +6. **Bind the application.** Confirm the `app_id` you used in step 4 is the + application you meant to talk to. The chain proves that the KMS issued this app + root key to *some* application; only this step ties it to yours. + +Step 2 is where a v0 verifier can go wrong in a way a v1 verifier cannot: the +claim is built from a `purpose` the verifier must already know out of band, and +from a hex string whose case and public-key encoding must match exactly. A +verifier that guesses `purpose`, uppercases the hex, or uses the uncompressed +SEC1 point computes a different digest and recovers a different, meaningless +public key in step 3, which then simply fails at step 5. + +### Test vector + +`get_key_pins_the_frozen_chain_link` in `dstack/guest-agent/src/rpc_service.rs` +pins link 0. With the fixture app root key `DUMMY_K256_KEY` from that test +module, + +```text +app root key = 1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b +path = "test" +purpose = "signing" +algorithm = "secp256k1" +``` + +`signature_chain[0]` is + +```text +c8a3dcf06c4e95bd78a5d7a1c8fcff171fc5848cfae804c6fc11bda4dc5d4062 +379995390843827444992c4c0e4bac70f0f878e01b9fc8b98cd7126fe5a3876b +01 +``` + +ECDSA here is deterministic (RFC 6979), so this vector pins the whole encoding +including the recovery byte. It is the only committed vector for a signature +chain on this surface; the values above are asserted on every test run. +[Report data](#report-data) has two of its own. + +## Sign + +`Sign` signs a caller-supplied payload with a key the agent derives itself. The +key is not chosen by the caller: `Sign` calls `GetKey` internally with +`path = "vms"` and `purpose = "signing"`, always, and only `algorithm` varies. +That is the same key `Worker.GetAttestationForAppKey` attests. + +`SignRequest.algorithm` goes through `normalize_algorithm` first, so `k256` is +accepted as an alias for `secp256k1` here too. Unlike `GetKey`, `Sign` has **no +default**: an empty `algorithm` derives a key and then fails with +`Unsupported algorithm`, because the empty string matches no signing mode. + +### Modes + +| `algorithm` | Key derived as | Bytes signed | Hash | Signature | +|---|---|---|---|---| +| `secp256k1` | secp256k1 at `vms`/`signing` | `data`, raw | SHA-256, by the agent | 64 bytes, `r \|\| s` | +| `k256` | alias for the row above | — | — | — | +| `secp256k1_prehashed` | secp256k1 at `vms`/`signing` | `data`, used **as the digest** | none | 64 bytes, `r \|\| s` | +| `ed25519` | ed25519 at `vms`/`signing` | `data`, raw | RFC 8032 (SHA-512, internal) | 64 bytes, `R \|\| S` | +| anything else | — | — | — | error, `Unsupported algorithm` | + +`r` and `s` are 32-byte big-endian integers and `s` is low-S normalised. Note the +shape: a payload signature is **64 bytes and carries no recovery byte**, unlike +the 65-byte chain links. It is the fixed-width form, not DER. + +`public_key` is the SEC1 compressed point (33 bytes) for both secp256k1 modes and +the RFC 8032 raw public key (32 bytes) for ed25519. + +The prehashed mode derives its key under the base name `secp256k1`, because +prehashing is a signing mode and not a key type. `secp256k1` and +`secp256k1_prehashed` therefore use the **same key** and produce the **same** +`public_key` and the same `signature_chain[1]` and `[2]`. + +Because `GetKey`'s derivation ignores the algorithm, the ed25519 and secp256k1 +modes also share one 32-byte secret, the one derived at `path = "vms"`. See +[One secret, two curves](#one-secret-two-curves). + +**`secp256k1_prehashed` puts the hash on the caller.** `data` must be exactly 32 +bytes; any other length fails with a message naming the length received. The +agent signs those 32 bytes directly as the digest and does not hash them again. +Consequences the caller owns: + +- Choose the hash function, and make sure the verifier uses the same one. The + agent records nothing about it, and neither does the signature. +- Domain-separate the payload before hashing. The agent will sign any 32 bytes, + including 32 bytes that are a digest of something else entirely, so a caller + that reuses one signing key across message types must build that distinction + into the preimage. +- The non-prehashed `secp256k1` mode hashes with SHA-256 inside the agent, so the + two modes produce different signatures over the same `data`, and a signature + from one does not verify under the other's rules. + +### The signature chain + +`SignResponse.signature_chain` has exactly three elements. The proto comment on +the field describes it accurately. + +```text +[0] the payload signature (64 bytes; the same bytes as `signature`) +[1] app root key signs "signing:" || hex(public_key) (65 bytes) +[2] KMS root key signs the app root public key (65 bytes) +``` + +Elements `[1]` and `[2]` are exactly `GetKey`'s `signature_chain[0]` and `[1]` +for a call with `path = "vms"`, `purpose = "signing"` and the mode's base +algorithm. Verify them by the steps in +[Verifying a chain](#verifying-a-chain), with `purpose = "signing"`. + +Element `[0]` duplicates the top-level `signature` field; it is there so that one +array carries the whole path from the payload to the KMS root. It is a different +shape from the other two: 64 bytes, no recovery byte, and verified against +`public_key` under the mode's own rules rather than recovered. + +## Verify + +`Verify` checks one signature against one public key. It is legacy: verification +needs no key material and no attestation, and the agent's answer arrives over the +socket unattested, so a relying party gains nothing over verifying locally. It is +retained because it is part of this frozen surface and the SDKs' v0 clients still +expose it. v1 has no counterpart. + +`algorithm` goes through `normalize_algorithm`, so the accepted values are +`ed25519`, `secp256k1`, `k256` (alias), and `secp256k1_prehashed`. An empty or +unrecognised value is an error, not `valid: false`. + +| `algorithm` | `public_key` | `data` | +|---|---|---| +| `ed25519` | RFC 8032 raw, 32 bytes | the message; hashed per RFC 8032 | +| `secp256k1` / `k256` | SEC1, compressed (33) or uncompressed (65) | the message; hashed with SHA-256 | +| `secp256k1_prehashed` | SEC1, compressed or uncompressed | the digest, used as-is | + +`signature` is the 64-byte `r || s` (or `R || S`) fixed-width form. DER is not +accepted. + +A malformed input is an error, not a verdict: a `public_key` that is not a valid +point, or a `signature` that is not 64 bytes or whose `r` or `s` is zero or at +least the group order, fails the call with HTTP 400. `valid: false` means the +inputs parsed and the signature did not check out. + +**A non-canonical high-S secp256k1 signature does not verify.** It parses, +because `r` and `s` are in range, and is then rejected by k256's verification, +so the call answers `valid: false` rather than failing. The effect is that a +malleated copy of a valid signature is not accepted here, which matters because +callers may be treating this answer as a uniqueness check. Verify the same way +if you replace this call with a local check. + +`secp256k1_prehashed` is more permissive here than in `Sign`, which is an +asymmetry worth knowing about. `Sign` requires `data` to be exactly 32 bytes. +`Verify` accepts any length from 16 bytes up: a shorter `data` is zero-padded on +the left to 32 and a longer one is truncated to its leftmost 32 bytes, both +silently. Below 16 bytes it is not an error either -- the length rejection is +swallowed and the answer is `200` with `valid: false`, like any other signature +that does not check out. Pass exactly the 32-byte digest and the two agree. + +A relying party that wants more than a single-signature check wants the chain +verification in [Verifying a chain](#verifying-a-chain), which this method does +not do: `Verify` never looks at a `signature_chain` and never touches a trust +anchor. + +## The remaining methods + +### GetTlsKey + +Issues a certificate and returns it with the key that backs it. The name is +misleading and v1 renamed it `IssueCert`: the operation is certificate issuance, +and the key is a by-product. + +**The key is freshly generated on every call, not derived.** No request field +feeds it, and two identical requests return two unrelated keys. It is a P-256 +key whose scalar comes from 32 bytes of `ring::rand::SystemRandom` output run +through the same HKDF-SHA256/`RATLS` step the derived keys use -- with a random +input, so the result is random and unreproducible either way -- returned as a +PKCS#8 PEM string in `key`. `GetKey` is the method that returns a stable, +re-derivable key. + +The agent builds a `CertSigningRequestV2` over the public key, signs the CSR with +the generated key, and relays it. With a KMS key provider the CSR goes to the KMS +`SignCert` RPC; without one it is signed by the local CA the app booted with. The +returned `certificate_chain` is whatever the signer produced. + +`usage_ra_tls` requests the RA-TLS quote extension, and `with_app_info` the app +info extension. `subject`, `alt_names`, `usage_server_auth` and +`usage_client_auth` map onto the certificate directly. + +`not_before` and `not_after` are seconds since the UNIX epoch and are optional +independently. When **both** are set, `not_before` must be strictly earlier than +`not_after`; otherwise the call fails. Setting only one is not validated here. +That check is one of 0.6.0's two sanctioned behaviour changes on this surface: a +0.5.x client that sent an inverted pair got a certificate; it now gets an error. + +### GetQuote + +Returns a raw Intel TDX quote over caller-supplied report data, with the event +log, the padded report data and the VM config. + +`report_data` is up to 64 bytes and is zero-padded on the right to 64. More than +64 bytes is an error, never a truncation. `GetQuoteResponse.report_data` is the +padded 64 bytes as the platform saw them. + +**As of 0.6.0 this method is Intel TDX only.** It used to answer on every +platform, returning an empty `quote`; a platform without a TDX quote now gets an +error naming `Attest` as the replacement. This is a behaviour change on a frozen +surface, and it is deliberate: an empty quote is not evidence, and a client that +treated it as one was already broken. See the 0.6.0 CHANGELOG entry beginning +"guest-agent: `GetQuote` is restricted to Intel TDX". + +On GCP Confidential VMs the gate passes, because there is a TDX quote, but the +answer carries the TDX half only. `GetQuoteResponse` has no field for the vTPM +quote GCP's own verification also binds, so a relying party there wants +`Attest`. + +`event_log` is a JSON array of TDX event log entries. V2 runtime events include +the hex-encoded preimage of their digest; a verifier should check that +`sha384(hex_decode(preimage))` equals the digest. + +### Attest + +Returns a `VersionedAttestation` over caller-supplied report data, as opaque +bytes in `attestation`. Report data padding is the same as `GetQuote`'s: up to 64 +bytes, zero-padded on the right, longer is an error. + +Unlike `GetQuote` it answers on every supported platform, and the attestation it +returns already carries the TDX quote and event log where those exist. The wire +format and how to extract a quote from it are specified in +[Extracting a quote from an attestation](./guest-api-v1.md#extracting-a-quote-from-an-attestation); +the format is shared between the two surfaces. + +`AttestResponse` on this surface carries `attestation` and nothing else. Field 2 +is `reserved`: it briefly carried `boottime_gpu_evidence` on an unreleased build. +Boot-time GPU evidence is a v1 feature, and a request sending +`include_boottime_gpu_evidence` here is ignored rather than honoured. + +### EmitEvent + +**Always fails, as of 0.6.0.** It answers HTTP 400 with + +```text +EmitEvent was removed in dstack 0.6.0; runtime RTMR3 events are system-owned and cannot be extended by apps +``` + +Runtime RTMR3 events became system-owned, so an application can no longer extend +the measurement chain. Applications bind their data through `report_data` +instead. + +The method was kept rather than deleted so that a pre-0.6 client gets a +self-explanatory error. Deleting it would answer HTTP 404 +`Service not found: EmitEvent`, which says nothing about why the events stopped +being recorded. The explanatory failure is the entire reason the method still +exists; `EmitEventArgs` is kept for the same reason, so an old client's request +still decodes and reaches the handler. + +### Info + +Returns application identity and configuration. It is not attestation: the +response arrives over a socket with no quote behind it, and nothing in it is +evidence. A relying party confirms these values against an attestation. + +`app_id`, `instance_id`, `device_id`, `mr_aggregated`, `os_image_hash` and +`compose_hash` are raw bytes (hex strings in JSON). `app_name`, +`key_provider_info` and `vm_config` are strings. `cloud_vendor` and +`cloud_product` are the DMI `sys_vendor` and `product_name` values, empty when +unreadable. + +`tcb_info` is a pretty-printed JSON string, not a message. It carries `mrtd` and +`rtmr0` through `rtmr3`, the full `event_log`, the verbatim `app_compose` +document, and hex copies of `mr_aggregated`, `os_image_hash`, `compose_hash` and +`device_id` that also appear as top-level fields. `compose_hash` is `sha256` over +the exact `app_compose` bytes, so do not parse and re-serialize before hashing. + +`app_cert` is a demo certificate the agent mints for a dashboard. It proves +nothing, it is requested lazily in the background, and it is an empty string +until that request completes. v1 dropped it. + +**What `public_tcbinfo` hides.** The gating applies only on the external +listener, and only when the app-compose does not set `public_tcbinfo`. In that +case `tcb_info` and `vm_config` come back as **empty strings**; every other +field, `key_provider_info` included, is served as normal. The internal +`DstackGuest.Info` applies no gating at all: the caller there is the application +itself, which cannot need protecting from its own configuration. + +Note that this is not what v1's `Worker.Info` does; v1 also blanks +`key_provider_info`. The difference is intentional and the frozen behaviour is +unchanged. See +[public_tcbinfo on the external surface](./guest-api-v1.md#public_tcbinfo-on-the-external-surface). + +### Version + +Returns the agent's `version` (its Cargo package version) and `rev` (the git +revision it was built from). It takes no arguments and touches nothing, which +makes it the cheapest liveness probe on either listener. + +### Worker.GetAttestationForAppKey + +On the external listener. Derives the application key for `algorithm`, builds +report data committing to its public key, and returns a `GetQuoteResponse` over +it. The caller cannot build that report data itself, because it does not know the +public key until the agent derives it. That is why attesting an app key needs +its own method rather than the caller-supplied `report_data` that `GetQuote` and +`Attest` take. + +The key is `GetKey(path = "vms", purpose = "signing", algorithm)`, the same key +`Sign` uses. `algorithm` goes through `normalize_algorithm`, and +`secp256k1_prehashed` derives under `secp256k1` exactly as it does in `Sign`. An +empty or unrecognised `algorithm` is an error. + +The report data is a DIP-1 tagged ASCII string, zero-padded on the right to 64 +bytes: + +```text +report_data = prefix || base64url_nopad(public_key) then right-padded with 0x00 to 64 +``` + +| `algorithm` | `prefix` | `public_key` | Total length | +|---|---|---|---| +| `secp256k1`, `k256`, `secp256k1_prehashed` | `dip1::secp256k1c-pk:` (20 bytes) | SEC1 compressed, 33 bytes | 20 + 44 = 64, no padding | +| `ed25519` | `dip1::ed25519-pk:` (17 bytes) | RFC 8032 raw, 32 bytes | 17 + 43 = 60, padded with 4 zero bytes | + +The base64 alphabet is URL-safe and unpadded. The construction fails rather than +truncating if it would exceed 64 bytes, so a commitment can never silently become +a valid-looking commitment to a different key. Note that the secp256k1 form fills +the field exactly: there is no trailing NUL to parse against, so a reader should +stop at 64 bytes or at the first NUL, whichever comes first. + +The `c` in `secp256k1c` records that the point is compressed. A verifier that +re-derives the commitment from an uncompressed point gets a different string. + +`app_key_report_data_matches_its_vectors` in +`dstack/guest-agent/src/rpc_service.rs` pins both forms against the fixture app +root key from [Test vector](#test-vector): + +```text +ed25519 "dip1::ed25519-pk:5Pbre1Amf1hrp2V2bbfKlIfxpQb2pJAmrgmhxgVoG9s\0\0\0\0" +secp256k1 "dip1::secp256k1c-pk:A6t_JdVkVdMAocH3f1f20WGT6JzdntxcXimUtEax8zc9" +``` + +`app_key_report_data_accepts_secp256k1_prehashed` asserts that +`secp256k1_prehashed` produces the second one unchanged. + +Because it returns a `GetQuoteResponse`, this method answers on Intel TDX and +fails everywhere else, exactly as `GetQuote` does. + +v1 ships no counterpart, on purpose: no v1 `GetKey(domain, algorithm)` can return +the key this attests, so a v1 method would hand back an attestation of a public +key whose private half the caller could not obtain. A v1 application attests its +own key instead. See +[There is no v1 AttestAppKey](./guest-api-v1.md#there-is-no-v1-attestappkey). + +## Tappd + +`Tappd` predates v0 and is deprecated. It is served on its own socket +(`/var/run/tappd.sock`) at `/prpc/`, with the `Tappd.` service-name prefix +stripped, and it is frozen by the same descriptor digest. New code should not +call it; the notes below exist so a reader of an old client can tell what it did. + +Every method maps onto a `DstackGuest` one, except where noted. + +| Method | Relationship to `DstackGuest` | +|---|---| +| `DeriveK256Key` | `GetKey`, verbatim. Same arguments, same bytes, fields renamed to `k256_key` and `k256_signature_chain` | +| `Info` | `Info`, with internal semantics: no `public_tcbinfo` gating | +| `Version` | `Version` | +| `TdxQuote` | `GetQuote`, but it hashes the report data first (see below) | +| `RawQuote` | `TdxQuote` with `hash_algorithm = "raw"`; requires exactly 64 bytes and does not pad | +| `DeriveKey` | *No `DstackGuest` equivalent.* See below | + +`TdxQuote` builds report data as `hash(prefix || ":" || content)` rather than +taking it raw. **The agent supplies the colon**; `prefix` is the tag alone. The +default is `app-data`, which the response echoes back without a colon, so +`hash_algorithm = "sha512"` over `content` digests `app-data:` + `content`. A +caller who passes `prefix = "app-data:"` because a quote is documented as +`hash("app-data:" + content)` gets `app-data::` + `content` and a quote nothing +will verify. The digest is left-aligned in the 64 bytes and the rest is zero. +`hash_algorithm = "raw"` passes `report_data` through unchanged and then requires +it to be exactly 64 bytes. + +`DeriveKey` is the one method with no counterpart: it derives a **P-256** key +pair at `path` and issues a certificate for it. Unlike `GetTlsKey`, whose key is +random, this key is derived from the app root key through the same +HKDF-SHA256/`RATLS` construction, so it is stable across calls, unless +`random_seed` is set, which replaces the app root key with 32 fresh random bytes +and makes the result unreproducible. It always requests `ext_app_info = false` +and takes no validity window. + +## Migration to v1 + +[`docs/guest-api-v1.md`](./guest-api-v1.md) is the current API. Its +[Field mapping](./guest-api-v1.md#field-mapping) table lists every v0 method and +field against its v1 counterpart, and +[Migration from the unversioned API](./guest-api-v1.md#migration-from-the-unversioned-api) +covers what changes for an application that moves. + +The one thing to read before doing anything else: **v1 derives different key +material.** Deriving under the same name on `/v1` returns different bytes than +`/v0` does, by design and with no compatibility mode. An application holding +assets or identity under a v0 key must migrate them deliberately rather than +switching URLs. + +Nothing forces a migration. This surface stays available and closed, and a +v0.5.x client keeps working against a 0.6 agent unchanged. + +## Related documents + +- [dstack Guest Agent API v1](./guest-api-v1.md), the current API and the + normative spec for `dstack.guest.v1` +- [Attestation on Intel TDX](./attestation-tdx.md) +- [App Compose format](./normalized-app-compose.md), for the document behind + `compose_hash` and `AppInfo.tcb_info`'s `app_compose` +- [On-chain governance](./onchain-governance.md), for the `DstackKms` contract + that publishes the KMS root public key used as the chain's trust anchor +- [`sdk/curl/api.md`](../sdk/curl/api.md), a curl-oriented tour of both surfaces diff --git a/docs/guest-api-v1.md b/docs/guest-api-v1.md new file mode 100644 index 000000000..c870ae00b --- /dev/null +++ b/docs/guest-api-v1.md @@ -0,0 +1,757 @@ +# dstack Guest Agent API v1 + +This is the normative specification of `dstack.guest.v1`, the versioned guest +agent API introduced in dstack 0.6.0. It defines the URL scheme, the key +derivation function, the signature chain encoding, and the steps a relying party +follows to verify what the agent produces. + +Read it as the contract. The proto file +(`dstack/guest-agent/rpc/proto/agent_rpc_v1.proto`) describes the message shapes; +this document pins the bytes. Where an implementation and this document disagree, +one of them is a bug. + +## What v1 is for + +v1 exists because the unversioned surface cannot be fixed without breaking the +v0.5.x clients that depend on it. That surface is closed at exactly the v0.5.11 +shape. v1 is where the corrections live. + +The guest agent is not a general-purpose crypto service. It holds two things no +caller can obtain elsewhere: the application root key, and the platform's ability +to attest. v1 serves those two things and nothing else. + +That is the organising principle behind the method set. `Sign` and `Verify` are +absent because they are pure computation over material the caller already has. +Any process that can reach this socket can ask `GetKey` for the private key +itself, so a server-side `Sign` grants no capability, adds an IPC round trip, and +adds an entry point to audit. Verification does not even need a key. + +Four things changed for the operations that remain. + +**Keys are domain-separated.** v0 derived a key from a path alone and handed the +same 32 bytes to both secp256k1 and ed25519. v1 binds the algorithm and a +versioned context tag into the derivation, so the two curves never share a secret +and a v1 key is never a v0 key. + +**A key is named by one field.** v0 had `path` and `purpose`, of which only `path` +reached the KDF while `purpose` was echoed into the chain claim. v1 merges them +into `domain`, and both `domain` and `algorithm` affect the derived key. + +**The chain claim is unambiguous.** v0's claim is a `:`-joined string over a +caller-chosen `purpose`, which lets an application steer what the app root key +signs. v1's claim is length-prefixed and binds raw public key bytes. + +**Names say what things are.** `GetTlsKey` became `IssueCert`, because the +operation is certificate issuance. `Info` no longer nests documents inside a JSON +blob called `tcb_info`. + +## Transport and versioning + +Every surface the agent serves is now a closed v0 fossil plus a v1. The +unversioned surfaces are exactly v0.5.11 and never change again; new capability +arrives only in `dstack.guest.v1`. + +The scheme is uniform: `v0` names the frozen v0.5.11 surface, `v1` the current +one, on both listeners. + +| Listener | Version | Service | Mount | Example path | +|---|---|---|---|---| +| Internal socket | v1 | `DstackGuest` | `/v1` | `/v1/GetKey` | +| Internal socket | v0 | `DstackGuest`, frozen | `/v0` | `/v0/GetKey` | +| Internal socket | v0 alias | `DstackGuest`, frozen | `/` | `/GetKey` | +| External | v1 | `Worker` | `/prpc/v1` | `/prpc/v1/Health` | +| External | v0 | `Worker`, frozen | `/prpc/v0` | `/prpc/v0/Info` | +| External | v0 alias | `Worker`, frozen | `/prpc` | `/prpc/Worker.Info` | + +The unversioned paths are compatibility aliases, kept so a pre-0.6 client keeps +working unchanged. They are additional mounts of the *same* handler, not a +parallel implementation, so they cannot drift from `/v0`. New code should say +which version it means. + +Both packages name their services `DstackGuest` and `Worker`. The version lives +in the package (`dstack.guest.v1`) and in the mount path, not in the service +name -- `dstack.guest.v1.DstackGuestV1` would say it twice. Where this document +needs to distinguish them it writes "the v1 `DstackGuest`" or "the frozen +`Worker`". + +`Tappd` is unchanged and outside this scheme: it predates v0 and stays on its own +socket at `/prpc/`. + +The internal socket is `/var/run/dstack.sock`, reachable only by the application +itself. The external listener is reachable by anyone who can route to the CVM. +That difference is the whole reason there are two services rather than one +mounted twice: the v1 `DstackGuest` hands out key material, and the v1 `Worker` +never does. + +Version selection is by URL path and nothing else. There is no header +negotiation, no `Accept-Version`, and no default-version redirect, so a request +URL is the complete record of which contract the caller asked for. An agent that +predates v1 has no `/v1` mount at all, so it answers `/v1/...` with a plain +HTTP 404 -- see [Detecting an agent without v1](#detecting-an-agent-without-v1). + +Both surfaces run over the same prpc transport. A `POST` carrying +`Content-Type: application/json` takes a JSON body and returns JSON; any other +content type takes a protobuf-encoded request message and returns a +protobuf-encoded response. A `GET` takes its fields as query parameters and +returns JSON. + +### Status codes + +| Status | Body | Meaning | +|---|---|---| +| 200 | the response message | Success | +| 404 | the server's own 404 page | No such mount: this agent has no surface at that path | +| 404 | `{"error": "Service not found: "}` | The surface is mounted; it has no such method | +| 400 | `{"error": ""}` | The method ran and failed | +| 501 | `{"error": ""}` | `AttestGpu` on an image that ships no GPU attestation | +| other | `{"error": ""}` | A handler chose the status; the message says why | + +A handler failure is a 400 with the error text in the body unless the handler +chose otherwise, and `AttestGpu` is the only v1 method that does. v1 does not +default, coerce, or truncate a malformed request into a well-formed one. + +### Detecting an agent without v1 + +Both "this agent is too old for v1" and "v1 exists but has no such method" +answer 404, so the status alone does not separate them. The body does. + +- 404 whose body is **not** JSON carrying `Service not found` -- there is no + surface mounted at that path. The agent predates v1. +- 404 whose body **is** `{"error": "Service not found: "}` -- v1 is + mounted and does not have that method. + +`/v1/Version` is the cheapest probe: it takes no arguments and touches nothing. + +## The internal surface + +The v1 `DstackGuest` has six methods. + +| Method | Purpose | +|---|---| +| `IssueCert` | Issue a certificate, with a freshly generated key | +| `GetKey` | Derive an application key and return it with its signature chain | +| `Attest` | Produce a versioned attestation over caller-supplied report data | +| `AttestGpu` | Collect GPU evidence now, against a caller-chosen nonce | +| `Info` | Return application identity and configuration | +| `Version` | Return the agent version | + +Five v0 methods are deliberately absent. + +`Sign` and `Verify` are absent for the reason above: neither needs the TEE. +Applications sign locally with a standard crypto library, using the key `GetKey` +returns, and verify locally following this document. A v0 client may keep calling +the unversioned `Verify` for single-signature checks; the v0 `Sign` RPC's +per-algorithm signing modes are specified in +[dstack Guest Agent API v0](./guest-api-v0.md#sign), which is the normative +document for that surface. + +`GetQuote` is absent because `Attest` subsumes it. `GetQuote` answers on Intel TDX +and nowhere else, and the `VersionedAttestation` that `Attest` returns already +carries the TDX quote and the event log. See +[Extracting a quote from an attestation](#extracting-a-quote-from-an-attestation). + +Boot-time GPU evidence has no method of its own. `Attest` with +`include_boottime_gpu_evidence` returns it, in the same `GpuEvidenceBundle` +shape `AttestGpu` uses, so one parser handles both and one round trip fetches +the evidence together with the attestation needed to authenticate it. + +`EmitEvent` is absent because runtime RTMR3 events became system-owned in 0.6.0. +An application binds its data through `report_data` instead. + +### Naming conventions + +Every request message is `Request` and every response is +`Response`, including for methods that take no arguments. An empty message +can gain a field later; `google.protobuf.Empty` cannot. + +A field's encoding lives in its doc comment, not in its name. Fields carrying JSON +documents say so and name who owns the schema. + +## Certificate issuance + +`IssueCert` is certificate issuance. The agent generates a key, builds a CSR, +signs the CSR with that key, and relays it to the KMS `SignCert` RPC, or to the +local CA when the application runs without a KMS. It returns the chain the signer +produced. + +v0 called this `GetTlsKey`, which named the by-product rather than the request. +The private key is incidental: it is freshly generated on every call, no request +field feeds it, and two identical requests produce two unrelated keys. `GetKey` is +the method that derives a stable, attestable key. + +`not_before` must be earlier than `not_after` when both are set; otherwise the +call fails. + +With `usage_ra_tls` set, the certificate carries the CVM's attestation in the +`PHALA_RATLS_ATTESTATION` extension (OID `1.3.6.1.4.1.62397.1.8`). v1 always puts +the MessagePack V1 schema there (see [Attestation](#attestation)), both in the CSR +and in the certificate issued from it, because the signer embeds the attestation +in the form it received. Two requirements follow: the KMS that signs the CSR must +be 0.5.9 or later, and so must any RA-TLS peer that verifies the certificate. +v0 `GetTlsKey` keeps the legacy SCALE form wherever it can represent the +attestation. + +This first cut serves only the integrated one-step mode, where the agent holds the +key. A mode that signs a caller-supplied CSR or public key, so the private key +never leaves the caller, is a plausible extension. It would arrive as added fields +or a sibling method, never as a change to what these fields mean. + +## Key derivation + +### Inputs + +A v1 application key is named by exactly two values. + +`domain` is a caller-chosen domain-separation string. It is not a DNS name; the +certificate fields on `IssueCertRequest` are the ones that take those. It may be +any byte string a proto3 `string` can carry, including one containing `:`, `/`, or +NUL. + +`algorithm` is `secp256k1` or `ed25519`. There is no default and no alias. An +empty string is an error, and so is any other value, including `k256` and +`secp256k1_prehashed`. + +Derivation is **flat**. Two domains yield unrelated keys. `a/b` is an opaque +string that happens to contain a slash, not a child of `a`, and no key derived +here can be used to derive another. There is no BIP-32-style hierarchy. The old +`path` name invited that reading, which is part of why it is gone. + +### Length-prefixed encoding + +Every construction below builds its input from length-prefixed fields. Write +`LP(x)` for: + +```text +LP(x) = uint32_be(len(x)) || x +``` + +`len(x)` is the byte length, encoded big-endian in exactly four bytes. Encoding +fails if a field is longer than 2^32 - 1 bytes. + +The prefix is what makes the encoding injective. A domain is arbitrary +caller-chosen bytes, so any delimiter it could also contain would let two +different `(domain, algorithm)` pairs encode to one byte string and share a key. +Joining with `:` or `/` is not sufficient here and is not what v1 does. + +### The KDF + +```text +salt = "dstack-guest-v1" (15 bytes, ASCII; v0 uses "RATLS") +IKM = app root secp256k1 private key (32 bytes, `k256_key` from .appkeys.json) +info = LP("dstack-guest-v1-key") || LP(algorithm) || LP(domain) +L = 32 + +key = HKDF-SHA256(salt, IKM, info, L) (RFC 5869: extract, then expand) +``` + +`algorithm` is bound as its canonical name, `secp256k1` or `ed25519`, never as the +string the caller sent. + +The primitive is unchanged from v0, which also used HKDF-SHA256 over the same +input key material. Two things changed. The `info` now binds the algorithm and a +version tag, where v0 passed the path alone, so the algorithm did not participate +and one 32-byte secret served both curves. And the salt is v1's own. + +The salt is what makes v1 a separate derivation tree rather than a +differently-labelled branch of the old one. Under a shared salt the two surfaces +would be separated only by their HKDF `info` -- and the legacy `info` is the +caller's `path` verbatim, so a caller that passed the v1 `info` byte string as a +v0 `path` would reproduce a v1 key exactly. Different salts close that by +construction, whatever either side puts in `info`. + +That collision was never a privilege boundary: both derivations serve the same +single-tenant application from the same root key, and that application may call +either surface. It is closed because a KDF whose separation depends on nobody +choosing an awkward input is one refactor away from not separating anything, and +the fix costs nothing on a surface with no deployed keys yet. + +The 32 output bytes are used directly: + +- **secp256k1**: the big-endian private scalar. If it is zero or at least the + group order, the call fails. That is a ~2^-128 event for a given domain, and the + caller can choose another one. Folding the scalar into range would silently land + two domains on one key. +- **ed25519**: the RFC 8032 seed, from which the key expands as usual. + +### Public key encoding + +| Algorithm | Encoding | Length | +|---|---|---| +| `secp256k1` | SEC1 compressed point, `0x02`/`0x03` prefix | 33 bytes | +| `ed25519` | RFC 8032 raw public key | 32 bytes | + +These are the exact bytes returned in `public_key` and the exact bytes bound into +the chain claim. A relying party never has to re-derive the public key from the +private key to check the chain. + +### Test vectors + +Generated with an app root key of + +```text +1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b +``` + +| domain | algorithm | private key | public key | +|---|---|---|---| +| `""` | secp256k1 | `59f60584ce6fd2a3a31997256db9d77322463fc8a6b1520110401bcb1ee92387` | `0377c7fb050db181d392266a3cee9adb2901c6d665f11bac68be5457f577ba4908` | +| `""` | ed25519 | `b023493030669cf22e9cafa6a464d4cf3ae4edfe5474ec796710f21ea011946d` | `a3dc149fd5b765eab2eb7d3174fa939e39386898f10b15b7b146f6f1358ecf2a` | +| `storage-encryption` | secp256k1 | `5510330f86902ddae38c6d89c93a8408019332c17a429e1abd01c4a28d1544a6` | `03d962450a41748021c8b02787ac36ce642ff0ae25f4c55019eb527e1112cfd764` | +| `storage-encryption` | ed25519 | `3c4c3ece12fa99ccb93fc0090877f80e70545fdd971e2ac93d3398c4684538d3` | `2380c4a33a60b60613fa43866421e5b96eb8dcde211317200fd0d41e7e491288` | +| `a/b/c` | secp256k1 | `7f0973449298085d2d36a3b4c4d3243c100ba1981ffa885fe9e9dee883e69538` | `02e9b1a61b6d70aa9b241753828c316bf90e33e77b2e113f9ba75a8b6dc3cde5c1` | +| `k\0:ey` | ed25519 | `42da8bf0b479ed125c370e3b91f982735bf08ff592abbd586985affa43ee96a1` | `c833107822b003ff5675b33b90b151d4315c3ab9162b17d876e8dffde41abf9b` | + +The last row's domain is the five bytes `6b 00 3a 65 79`. It is there because a +delimiter-joined encoding would mishandle it. + +The same vectors are asserted in `dstack/ra-tls/src/api_v1.rs`, which pins the +private-key column next to the primitives that produce it; +`dstack/guest-agent/src/rpc_service_v1/keys.rs` pins the public-key column +through the type the handler actually serves. They describe deployed key +material, so a diff against them is a bug in the derivation, not a stale fixture. + +## The signature chain + +`GetKey` returns a two-element `signature_chain` alongside the key. + +```text +[0] app root key signs the v1 key claim (specified here) +[1] KMS root key signs the app root public key (unchanged from v0) +``` + +Link 1 is produced by the KMS, outside the agent, and both API surfaces pass the +same bytes through. Only link 0 is new. + +### Link 0: the key claim + +```text +claim = LP("dstack-guest-v1-key-claim") + || LP(algorithm) + || LP(domain) + || LP(public_key) + +digest = keccak256(claim) +link0 = r || s || v (65 bytes) +``` + +`link0` is a recoverable ECDSA secp256k1 signature over `digest` by the app root +key. `r` and `s` are 32-byte big-endian integers, low-S normalised, and `v` is the +one-byte recovery id in `0..=3`. `public_key` is the raw derived public key from +the table above, not a hex string. + +A worked claim, for `domain = "storage-encryption"` and `algorithm = "secp256k1"`: + +```text +00 00 00 19 "dstack-guest-v1-key-claim" 25 bytes +00 00 00 09 "secp256k1" 9 bytes +00 00 00 12 "storage-encryption" 18 bytes +00 00 00 21 03 d9 62 45 ... d7 64 33 bytes +``` + +With the app root key from the test vector table, `link0` is + +```text +5b6193729ce7976ec67863f21692d4b98c69832698aae8e001a7d33a6f818b6e +46ca950725b6e90e8ca9bcf394abd03ce264bf9b7eec1e91693247f9dd53c269 +01 +``` + +The signature is deterministic (RFC 6979), so this vector pins the whole encoding +including the recovery byte. + +#### Why this cannot be forged through v0 + +v0's claim is `keccak256("{purpose}:{hex(pubkey)}")`, and `purpose` is an +arbitrary caller-supplied string. A malicious application can therefore make the +app root key sign nearly any ASCII byte string that ends in `:` followed by +lowercase hex. If a v1 claim were reachable that way, the v1 chain would be worth +nothing. + +It is not reachable, and the reason is structural rather than probabilistic. + +Every v0 preimage ends with `:` followed by the hex encoding of a public key, so +its last 64 bytes (ed25519) or 66 bytes (secp256k1) are all lowercase hex +characters. A v1 claim ends with `LP(public_key)`, whose four length bytes are +`00 00 00 21` for secp256k1 and `00 00 00 20` for ed25519, sitting 37 or 36 bytes +from the end. That is inside the region a v0 preimage requires to be hex-only, and +`0x00` is not a hex character. No choice of `purpose` reproduces a v1 claim byte +string, and since the two byte strings can never be equal, matching their keccak +digests would require a preimage attack on keccak256. + +The regression test is `a_v0_claim_cannot_be_crafted_into_a_v1_claim` in +`dstack/ra-tls/src/api_v1.rs`. It builds the strongest available forgery, a +`purpose` set to the v1 claim minus exactly the suffix v0 appends on its own, +then asserts both that the byte strings differ and that the structural property +holds. + +The two context tags are also distinct, so a derivation input can never be read as +a claim: `dstack-guest-v1-key` for the KDF, `dstack-guest-v1-key-claim` for the +claim. Because both are length-prefixed, neither encoding is a prefix of the other. + +### Link 1: the KMS attestation + +Unchanged from v0 and reproduced here so this document is self-contained. + +```text +message = "dstack-kms-issued" || ":" || app_id || sec1_compressed(app_root_pubkey) +digest = keccak256(message) +link1 = r || s || v (65 bytes) +``` + +`link1` is a recoverable ECDSA secp256k1 signature over `digest` by the KMS root +key. `app_id` is the raw app id bytes, and the app root public key is SEC1 +compressed, 33 bytes. + +### Verifying a chain + +A relying party holds a `public_key`, a `signature_chain`, the `(domain, +algorithm)` the key was requested under, and the `app_id`. It performs the +following. + +1. **Anchor.** Obtain the KMS root public key from a source you trust + independently of the agent being checked: the `DstackKms` contract's + `kmsInfo().k256Pubkey`, or a value pinned out of band. This step carries the + security of everything below it. An attacker who can answer your query for the + anchor can also mint a self-consistent chain, so reading the anchor from the + KMS you are checking proves nothing. + +2. **Rebuild the claim.** Compute + `claim = LP("dstack-guest-v1-key-claim") || LP(algorithm) || LP(domain) || LP(public_key)` + using the canonical algorithm name and the raw public key bytes, then + `digest0 = keccak256(claim)`. + +3. **Recover the app root key.** Split `signature_chain[0]` into `r`, `s`, `v` and + recover the secp256k1 public key from `(digest0, r, s, v)`. Reject a + non-canonical high-S `s`. Call the result `app_root_pubkey`, SEC1 compressed. + +4. **Rebuild the KMS message.** Compute + `digest1 = keccak256("dstack-kms-issued" || ":" || app_id || app_root_pubkey)`. + +5. **Check link 1.** Verify `signature_chain[1]` over `digest1` against the anchor + from step 1, either by recovering and comparing to the anchor or by verifying + `(r, s)` against it directly. Reject high-S here too. + +6. **Bind the application.** Confirm the `app_id` you used in step 4 is the + application you meant to talk to. The chain proves that the KMS issued this app + root key to *some* application; only this step ties it to yours. + +To also check a payload signature an application produced with a key from +`GetKey`, verify that signature against `public_key` under whatever scheme the +application used, then run steps 2 through 6 to establish that `public_key` is what +it claims to be. The two halves are independent: a valid payload signature under an +unverified public key says nothing. + +Step 3 recovers the app root key rather than requiring it as an input, which is why +link 0 carries a recovery byte. A verifier that already knows the expected app root +public key may verify `(r, s)` against it directly and compare instead. + +## The external surface + +The v1 `Worker` has three methods, served at `/prpc/v1`. + +| Method | Purpose | +|---|---| +| `Info` | Application identity and configuration, subject to `public_tcbinfo` | +| `Version` | The agent version | +| `Health` | Report whether the application is serving | + +Nothing here returns key material, and no caller chooses what gets signed or +attested. + +### There is no v1 AttestAppKey + +The frozen surface has `Worker.GetAttestationForAppKey`, which attests a key the +agent derives from an algorithm name alone. v1 has no counterpart, on purpose. + +That method attests the key v0's KDF derives at path `vms` with purpose +`signing`. No v1 `GetKey(domain, algorithm)` call can return that key: the v1 +KDF has a different salt, a different `info`, and no `purpose` input at all. A +v1 application calling it would receive an attestation of a public key whose +private half it has no way to obtain, which is worse than having no method -- +it looks like it works. + +A v1 application attests its own key instead: + +1. Derive the key: `GetKey(domain, algorithm)` on the internal socket. +2. Commit to it: build `report_data` over `public_key` yourself. Prefix it so a + verifier can tell what it is looking at -- see the `dip1::` convention the + frozen method uses. +3. Attest it: `Attest(report_data)` on the internal socket. +4. Publish it: hand the resulting attestation to relying parties. The + application serves it; the agent's external listener does not. + +This is strictly more capable than the method it replaces. The application picks +which of its keys to attest, and picks the commitment format, instead of being +limited to the single key the agent would derive for it. + +Legacy flows keep using `Worker.GetAttestationForAppKey`, unchanged and frozen. +It is Intel TDX only, because it returns a `GetQuoteResponse`. + +`Health` is polled by the gateway to decide whether an instance belongs in its +application's load-balancing rotation. It answers from a cache the agent +refreshes on its own timer, so one call costs a lock and a clone however many +gateway nodes are polling. Only instances that opted in via +`RegisterCvmRequest.health_check` are ever polled; see +[Application health checks](./app-health-checks.md). + +An app that never opted in answers `healthy: true` with an empty `unhealthy` +and no `error`, whatever its containers are doing: it declared itself +un-polled, so the agent computes no verdict and returns what the gateway would +have assumed. A caller other than the gateway must read that `true` as "nobody +asked me to know", not as a health check that passed. + +### public_tcbinfo on the external surface + +The v1 `Worker.Info` returns the same `InfoResponse` the internal surface returns, +minus what the application asked to keep private. Unless the app-compose sets +`public_tcbinfo`, the three document fields come back as empty strings: + +- `app_compose` +- `vm_config` +- `key_provider_info` + +Identity and the measurement hashes are always present. + +This is close to, but deliberately not the same as, what the frozen +`Worker.Info` does. That one blanks `tcb_info` and `vm_config`, and serves +`key_provider_info` externally in every case. v1 blanks `key_provider_info` too: +it names the component holding the application's keys, and an external caller +has no use for it. The frozen behaviour is unchanged on its own surface, so a +v0.5.x client sees exactly what it always did. + +| Field | frozen `Worker.Info` | v1 `Worker.Info` | +|---|---|---| +| identity, measurement hashes | always served | always served | +| `tcb_info` / measurement registers | blanked | not in the message at all | +| `vm_config` | blanked | blanked | +| `app_compose` | (nested in `tcb_info`, blanked) | blanked | +| `key_provider_info` | **always served** | **blanked** | + +The internal v1 `DstackGuest.Info` applies no gating at all. The flag decides what +an outside party may learn, and the caller on the internal socket is the +application itself; an application cannot need protecting from its own +configuration. + +## Attestation + +`Attest` is v1's only CVM attestation entry point. It takes up to 64 bytes of +`report_data`, zero-padded on the right to 64; more than 64 bytes is an error +rather than a truncation. It returns the versioned dstack attestation format, +which covers every supported platform. + +### Extracting a quote from an attestation + +`AttestResponse.attestation` is a `VersionedAttestation`. The authoritative +implementation is `dstack/dstack-attest/src/v1.rs`, and `dstack-verifier` is the +reference consumer. + +v1 `Attest` always returns the V1 schema: a MessagePack map produced by +`rmp_serde::to_vec_named`, whatever platform the CVM runs on and whatever +runtime event-log version it uses: + +```text +{ "version": u64, + "platform": { "kind": , "data": { ... } }, + "stack": { "kind": , "data": { ... } } } +``` + +`platform.kind` is one of `tdx`, `gcp-tdx`, `nitro-enclave`, `aws-nitro-tpm`, or +`sev-snp`. For `tdx` and `gcp-tdx`, `platform.data` carries `quote` (the raw TDX +quote bytes, exactly what the unversioned `GetQuote` returned) and `event_log`. +`gcp-tdx` additionally carries `tpm_quote`, which GCP's own verification binds and +which `GetQuote` had no field for. The other three platforms have no TDX quote, +which is why `GetQuote` could not answer on them at all. + +`stack.data.report_data` carries the 64 bytes the caller asked for. + +A general `VersionedAttestation` decoder still has to accept the legacy form, in +which the first byte is `0x00` and the rest is SCALE-encoded V0. The frozen v0 +`Attest` and `GetTlsKey` still produce it, and so do the KMS and gateway +certificates built on them and every older release; a decoder tells the two +forms apart by that first byte. A client that reads only v1 `Attest` can treat +anything other than a MessagePack map as malformed. MessagePack needs dstack +0.5.9 or later on the verifying side (`dstack-verifier`, KMS, gateway); earlier +releases decode only SCALE. + +V2 runtime events in the event log always include the hex-encoded preimage of +their digest; a verifier should check that `sha384(hex_decode(preimage))` equals +the digest. + +### GPU evidence + +Both GPU methods return the same thing: a list of `GpuEvidenceBundle`, each +carrying `vendor`, `format`, and opaque `evidence` bytes. `AttestGpu` returns +its bundles directly; `Attest` returns them in +`AttestResponse.boottime_gpu_evidence`. A consumer writes one bundle parser and +dispatches on `(vendor, format)`. + +| Source | `vendor` | `format` | Answers | +|---|---|---|---| +| `AttestGpu` | `nvidia` | `nvidia-nvattest-collect-evidence-json-v1` | is this device genuine right now | +| `Attest` | `nvidia` | `nvidia-nvattest-boottime-json-v1` | what the boot looked like | + +The two `format` values are deliberately distinct. They answer different +questions and a verifier for one does not appraise the other, so sharing a tag +would leave a consumer no way to tell a live measurement from a boot record. + +`boottime_gpu_evidence` is populated only when the request sets +`include_boottime_gpu_evidence` and boot-time output exists. Absence is the +empty list; there is no sentinel value. + +Its `evidence` is the exact bytes of the nvattest output as the agent read them +from disk. That exactness is the contract, not an implementation detail: the +only thing tying this evidence to the boot is `sha256` over precisely those +bytes, compared against the `evidence_sha256` field of the measured +`gpu-attestation` event after replaying the runtime event log. Parsing and +re-serializing the JSON before hashing changes the digest and breaks the +comparison. + +That evidence is not bound to `report_data` -- nvattest ran at boot against its +own nonce, so a fresh `report_data` says nothing about it. + +It is also a historical statement about the boot, not a live one: it does not +prove the GPU is still attached. Sampling the GPU at attestation time would not +fix it, because an NVIDIA report binds the device and a nonce but not the TD the +device is attached to, so a fresh report can be relayed from a genuine remote +GPU. Only TDISP/TEE-IO device binding closes that gap. + +`AttestGpu` answers the narrower question of whether the device reachable right +now is a genuine CC-enabled GPU that signs a caller-chosen 32-byte nonce. It +returns vendor-native evidence rather than a local verdict, so a relying party +appraises it with its own verifier: select one by `vendor` and `format`, then +check the signature, certificate chain, measurements, and the nonce embedded in +the evidence. + +## Info + +`Info` returns identity and configuration. It is not attestation, and nothing it +returns is evidence: the response arrives over a local socket with no quote behind +it. + +That is why the measurement registers and the event log are absent. v0's +`AppInfo.tcb_info` carried MRTD, RTMR0-3 and the event log inside a JSON string, +which invited relying parties to read measurements out of an unattested response. +Those values belong to `Attest`, whose attestation carries them quote-backed. + +`mr_aggregated`, `os_image_hash` and `compose_hash` remain, deliberately. They +identify *which* application and image this is, which is the question `Info` +answers. They are typed bytes rather than hex strings inside a JSON blob, and each +appears exactly once; v0 returned all three both as top-level fields and again, +hex-encoded, inside `tcb_info`. They are still unattested, and a relying party +still confirms them against an attestation. + +`app_cert` is gone. It was a self-issued demo certificate the agent minted for a +dashboard, and it proved nothing. + +`app_compose` carries the verbatim deployed document. `compose_hash` is `sha256` +over exactly those bytes, so do not parse and re-serialize before hashing: key +order, whitespace and unknown fields all change the digest, and that digest is +what gets whitelisted on chain. + +The v1 `DstackGuest.Info` on the internal socket applies no hiding: the caller is the +application itself, which cannot need protecting from its own configuration. +The v1 `Worker.Info` on the external listener honours `public_tcbinfo`; see +[public_tcbinfo on the external surface](#public_tcbinfo-on-the-external-surface), +which is the authoritative description. + +## Errors + +| Condition | Behaviour | +|---|---| +| Empty or unrecognised `algorithm` | Error naming the accepted values | +| Derived secp256k1 scalar out of range | Error; the caller picks another domain | +| `report_data` longer than 64 bytes | Error | +| `nonce` not exactly 32 bytes | Error naming the required length | +| GPU attestation unavailable in this image | HTTP 501; `AttestGpu` has nothing to collect | +| `not_before` not earlier than `not_after` | Error | +| Unknown method on a mounted surface | HTTP 404, `Service not found: ` | +| `/v1/...` on an agent that predates v1 | HTTP 404, no such mount | + +Every row that does not name a status is an HTTP 400 with the message in the +body. See [Status codes](#status-codes) for the full mapping. v1 does not +default, coerce, or truncate a malformed request into a well-formed one. + +The GPU row is a 501 rather than a 400 because it is a property of the image, +not of the call: the request is well-formed, and no other request would succeed +either. A client that reads 400 there retries with different arguments forever; +one that reads 501 stops and falls back to whatever it does without a GPU. A +malformed nonce is still the caller's fault and still a 400. + +## Migration from the unversioned API + +**v1 keys are different keys.** Deriving under the same name on `/v1` that an +application used on `/` returns different key material. This is the point of the +new KDF, not a defect: the v0 derivation ignored the algorithm, so one secret +served two curves. There is no compatibility mode and no flag to get the old bytes +back from `/v1`. + +An application holding assets or identity under a v0 key must migrate them +deliberately. Derive the v1 key, move the asset with a transaction signed by the +v0 key, and only then cut over. An application with no persistent state can switch +by pointing at the new URL. + +Both unversioned surfaces stay available and closed, and they have their own +normative specification: [dstack Guest Agent API v0](./guest-api-v0.md) pins the +frozen KDF, the `:`-joined chain claim, and the per-algorithm `Sign` and `Verify` +modes. A v0.5.x client keeps working against a 0.6 agent with no changes: `Sign`, `Verify` and `EmitEvent` +remain on the internal one (`EmitEvent` fails with a message naming its removal), +and `GetAttestationForAppKey` remains on the external one. Nothing forces a +migration. + +**SDK shape.** All four SDKs ship two clients mirroring the two surfaces, and +**the unsuffixed client is this one**: `DstackClient` names the v1 client and is +the recommended default. `DstackClientV0` is the closed unversioned API, +explicitly named and marked legacy, and still carries its `Sign` and `Verify` +RPCs. They are transport mirrors, not a compatibility layer: neither translates +a call to the other, and each one's method set is exactly its surface's. + +Every field this document declares `bytes` is that language's byte type in the +v1 clients -- `Vec`, `bytes`, `[]byte`, `Uint8Array` -- with hex confined to +serialization. `public_key` is why: the claim above is built over the raw key, +and a hex string handed to a claim builder silently produces a chain that never +verifies. The v0 clients keep their hex strings and `decode_*` helpers, because +that surface is frozen. + +That alias flipped in 0.6.0. Code that used the unsuffixed client for v0 calls +fails loudly on upgrade -- the v1 signatures differ and `GetKey` requires +`algorithm` explicitly -- rather than silently deriving different keys under the +new KDF. To stay on the frozen surface, name `DstackClientV0`. + +`ClientV1` has no `Sign` and no `Verify`, because v1 has neither. An application +signs locally with the key `GetKey` returns, and a relying party verifies +locally, following the rules above. The SDKs deliberately ship no verification +helper: verifying needs no client and no connection, and this document is what a +verifier implements against. + +## Field mapping + +For readers porting from the unversioned API. + +| v0 | v1 | Note | +|---|---|---| +| `GetTlsKey` | `IssueCert` | Renamed; the embedded RA-TLS attestation is always MessagePack V1 | +| `GetKeyArgs.path` + `.purpose` | `GetKeyRequest.domain` | Merged; both KDF inputs now | +| `GetKeyArgs.algorithm` (defaulted) | `GetKeyRequest.algorithm` | Required; no `k256` alias | +| — | `GetKeyResponse.public_key` | Added | +| `GetQuote` | `Attest` | TDX-only channel subsumed | +| `AppInfo.tcb_info` | — | Measurements are typed fields; the rest belongs to `Attest` | +| `AppInfo.app_cert` | — | Dashboard artifact | +| `AppInfo.vm_config` | `InfoResponse.vm_config` | Unchanged content | +| `AppInfo.key_provider_info` | `InfoResponse.key_provider_info` | Unchanged content | +| (nested in `tcb_info`) | `InfoResponse.app_compose` | Promoted to top level | +| `Sign` | — | Removed; sign locally with the key from `GetKey` | +| `Verify` | — | Removed; verify locally per this document | +| `EmitEvent` | — | Removed; RTMR3 is system-owned | +| `Worker.GetAttestationForAppKey` | — | No v1 counterpart; a v1 app attests its own key, see above | +| frozen `Worker.Info` | v1 `Worker.Info` | Also gates `key_provider_info`; see above | + +## Related documents + +- [dstack Guest Agent API v0](./guest-api-v0.md), the normative spec for the + frozen unversioned surface +- [Attestation on Intel TDX](./attestation-tdx.md) +- [Application health checks](./app-health-checks.md), for `/prpc/v1/Health` +- [App Compose format](./normalized-app-compose.md), for the schema behind + `InfoResponse.app_compose` +- [On-chain governance](./onchain-governance.md), for the `DstackKms` contract that + publishes the KMS root public key diff --git a/docs/guest-netfilter-capabilities.md b/docs/guest-netfilter-capabilities.md new file mode 100644 index 000000000..ce7f3bc5e --- /dev/null +++ b/docs/guest-netfilter-capabilities.md @@ -0,0 +1,211 @@ +# Guest netfilter capabilities + +Which packet-filtering capabilities a dstack guest image provides, and which +firewall path an application running nested container managers (Incus, LXD, +libvirt, a nested Docker) should expect to work. + +Applications that only publish ports through docker-compose do not need this +page: dstack's own networking and Docker's published ports work on every image. +It matters when something inside the CVM programs its own bridges and firewall +rules. + +## One dataplane: nftables + +Every netfilter frontend in the image writes to nftables. `iptables`, +`ip6tables` and `ebtables` are the nft-backed frontends, so Docker's rules, +dstack's own `DSTACK_WG` chain and anything a tenant workload writes all land +in one ruleset that a single `nft list ruleset` can show. + +This matters more than tidiness. The legacy and nf_tables rulesets register at +the same netfilter hooks but cannot see each other, so on a mixed image the +effective ruleset is only knowable by querying both. A container manager that +selects a firewall driver by probing which ruleset is already in use gets +steered by whatever Docker happened to do — which is what made Incus managed +bridges fail on 0.5.x, where the frontends were legacy. + +Both backends reach this the same way, but from different directions: + +| | `os/yocto` | `os/mkosi` | +| --- | --- | --- | +| Frontend | `xtables-nft-multi`, linked in `iptables_%.bbappend` | Debian's nft frontend | +| How it is guaranteed | built explicitly by the recipe | asserted in `parity.json` | +| `NETFILTER_XTABLES_LEGACY` in kernel | `y` (from `netfilter.scc`) | not set | +| Legacy frontends still present | yes, as `*-legacy` | binaries exist, no kernel tables | + +Neither image *removes* the legacy binaries, so `iptables-legacy` remains +callable. On Yocto it still works, because that kernel keeps the legacy tables; +on mkosi it has no tables behind it. Use the unsuffixed commands. + +## Capability matrix + +Everything listed is a loadable module (`=m`) unless stated. Values were read +from a built `kernel-config` artifact for Yocto, and from the `.config` that +`x86_64_defconfig` plus `os/mkosi/components/kernel/kernel.config` produces for +mkosi. + +| Capability | Kconfig symbol | Yocto | mkosi | Since | +| --- | --- | --- | --- | --- | +| nftables core | `NF_TABLES` | yes | yes | 0.5.x | +| xtables-over-nftables | `NFT_COMPAT` | yes | yes | 0.5.x | +| nftables bridge family | `NF_TABLES_BRIDGE` | yes | yes | 0.6.0 (mkosi) | +| bridge meta / reject | `NFT_BRIDGE_META`, `NFT_BRIDGE_REJECT` | yes | yes | 0.6.0 | +| ebtables framework | `BRIDGE_NF_EBTABLES` | yes | yes | 0.5.x | +| ebtables matches | `BRIDGE_EBT_ARP`, `_IP`, `_IP6`, `_AMONG`, `_LIMIT`, `_VLAN` | yes | yes | 0.6.0 | +| ebtables legacy tables | `BRIDGE_NF_EBTABLES_LEGACY`, `BRIDGE_EBT_T_*` | **no** | **no** | — | +| CHECKSUM target | `NETFILTER_XT_TARGET_CHECKSUM` | yes | yes | 0.6.0 (mkosi) | +| IPv4 tables | `IP_NF_IPTABLES` | yes | built in | 0.5.x | +| IPv4 legacy tables | `IP_NF_IPTABLES_LEGACY` | yes | **no** | — | +| IPv6 tables | `IP6_NF_IPTABLES` | yes | built in | 0.6.0 (Yocto) | +| IPv6 NAT / filter / mangle | `IP6_NF_NAT`, `IP6_NF_FILTER`, `IP6_NF_MANGLE` | yes | n/a | 0.6.0 (Yocto) | +| ipset | `IP_SET` and the hash/bitmap set types | yes | yes | 0.5.x | + +Userspace: both images ship `iptables` and `nftables`. `ebtables` is the +nft-backed frontend from the same multi-call binary, not the standalone legacy +package. + +Three entries deserve a note. + +**The legacy ebtables tables are deliberately absent.** +`BRIDGE_NF_EBTABLES` builds only the matches, targets and watchers; since the +6.15 legacy-tables split the tables and `ebtables.ko` itself hang off +`BRIDGE_NF_EBTABLES_LEGACY`, which defaults to `n`. Nothing on an nft frontend +uses them — `ebtables-nft` emits nft bridge rules and reaches the `ebt_*` +match modules through `nft_compat`. + +**The IPv6 legacy tables on Yocto are for completeness, not for the default +path.** The nft frontend synthesises whatever table it is asked for, so +`ip6tables` needs no `ip6table_nat` module. They are enabled because that +kernel keeps `NETFILTER_XTABLES_LEGACY` on and has the full IPv4 legacy set, so +IPv6 being the one missing family meant `ip6tables-legacy` could not even list +a table. + +**`CHECKSUM` is still required.** The DHCP path of a managed bridge appends +`-j CHECKSUM --checksum-fill` so dnsmasq's replies reach guests with +checksum-offloading virtio NICs. On an nft frontend that is programmed through +`nft_compat`, which still needs the `xt_CHECKSUM` module. + +## Beyond netfilter: traffic control and checkpoint/restore + +Two more guest-kernel capabilities a nested container manager depends on live +in the same two fragments. Both are built in (`=y`), so neither backend has a +module to package or autoload. + +| Capability | Kconfig symbol | Yocto | mkosi | Since | +| --- | --- | --- | --- | --- | +| HTB qdisc | `NET_SCH_HTB` | built in | built in | 0.6.0 | +| ingress qdisc | `NET_SCH_INGRESS` | built in | built in | 0.6.0 | +| u32 classifier | `NET_CLS_U32` | built in | built in | 0.6.0 | +| police action | `NET_ACT_POLICE` | built in | built in | 0.6.0 | +| checkpoint/restore | `CHECKPOINT_RESTORE` | built in | built in | 0.6.0 (mkosi) | +| macvlan | `MACVLAN` | built in | built in | 0.6.0 (mkosi) | +| comment match | `NETFILTER_XT_MATCH_COMMENT` | yes | yes | 0.6.0 (mkosi) | +| socket diag | `UNIX_DIAG`, `INET_DIAG`, `PACKET_DIAG`, `NETLINK_DIAG` | built in | built in | 0.6.0 (mkosi) | + +**Per-instance bandwidth limits need all four `tc` pieces.** Incus implements +`limits.max` (and `limits.ingress` / `limits.egress`) on a bridged NIC as an +HTB root qdisc and class on the host-side veth for egress, and an ingress +qdisc with a u32 filter and a police action for ingress. `NET_SCHED` and +`NET_CLS_ACT` alone do not provide any of them: on 0.5.x the manager accepts +the setting and then fails to start the instance with `Specified qdisc kind is +unknown` (#1176). + +**`CHECKPOINT_RESTORE` is what lets the LXC monitor be found.** LXC retitles +its monitor process to `[lxc monitor] ` with +`prctl(PR_SET_MM, PR_SET_MM_MAP, ...)`, and Incus's seccomp notification +handler resolves a monitor PID to its instance by that title. The prctl is +only implemented under `CHECKPOINT_RESTORE`; without it the monitor keeps its +`incusd forkstart` command line, the daemon logs `Failed to find container +for monitor `, and every intercepted syscall +(`security.syscalls.intercept.*`) is resumed unhandled instead of getting the +container-aware answer (#1180). Everything the symbol gates needs +`CAP_CHECKPOINT_RESTORE` or `CAP_SYS_ADMIN`, and none of it faces the host. +The Yocto 6.18 kernel already had it through meta-virtualization's +`cfg/lxc.scc` and `cfg/criu.scc`; the mkosi kernel did not, and the 6.9 +kernel of 0.5.9 did not either. + +**LXC's own audit runs against both kernels.** `lxc-checkconfig` is the +script LXC ships to check a kernel for what containers need, and it is the +closest thing Incus has to Docker's `check-config.sh` — Incus itself has no +equivalent, and its documentation defers to "any kernel feature required by +the LXC version in use". It is vendored at `os/common/scripts/lxc-checkconfig` +and `check-lxc-kernel-config.sh` runs it after the fragment gate in both +builds, failing on anything it reports missing. The last three rows above are +what it flagged on the mkosi kernel the first time it ran: `MACVLAN` (Incus +`nictype=macvlan`), `xt_comment` (`-m comment` on the xtables frontends) and +the socket-diag interfaces `ss` and CRIU use. It does not know about traffic +control, so the `tc` rows are asserted by the fragments alone. + +## Running a nested bridge manager alongside Docker + +Having the capabilities is not the whole story. Docker sets the `FORWARD` chain +policy to `drop`, and `br_netfilter` is loaded with +`net.bridge.bridge-nf-call-iptables = 1`, so frames bridged between two +containers on someone else's bridge still traverse that chain. nftables base +chains in different tables are evaluated independently at the same hook, so a +manager that installs its own `accept` rules in its own table — Incus creates +`table inet incus` with `policy accept` and per-bridge accept rules — does not +override Docker's `drop`. The bridge comes up, DHCP works, containers get +addresses, and then nothing can talk to anything. + +This is the long-standing interaction between Docker's forward policy and +libvirt/LXD/Incus-style bridge managers, and it is what every distribution that +defaults to the nftables frontend already behaves like. + +It is, however, a behaviour change for the Yocto image, which programmed the +legacy tables before dstack 0.6.0. On that path Incus selected its xtables +driver and installed its accept rules with +`iptables -I filter FORWARD -i -j ACCEPT` — into Docker's *own* chain, +ahead of the policy — so forwarding worked without any extra rule. Under the +nftables driver the accept lands in a separate table and no longer pre-empts +Docker's policy. In practice the older image could not create a working managed +bridge at all (that is what #1032 reports), so few deployments will have relied +on it, but a setup that used the reported workaround of disabling DHCP and IPv6 +will now need the rule below. + +One rule per managed bridge fixes it: + +```sh +iptables -w 5 -I DOCKER-USER -i -o -j ACCEPT +ip6tables -w 5 -I DOCKER-USER -i -o -j ACCEPT +``` + +Measured in a CVM with Incus 6.0.4 and two system containers on a managed +bridge: 100% packet loss before, 0% after, for both IPv4 and IPv6. + +One benefit of the single nftables dataplane is that this is now diagnosable. +`nft list ruleset` shows Docker's `policy drop` and Incus's accept rules +together; when the two lived in different rulesets, neither `iptables -L` nor +`nft list ruleset` alone showed both halves of the interaction. + +## Adding a capability + +Kernel configuration lives in two fragments, one per backend: + +- `os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg` +- `os/mkosi/components/kernel/kernel.config` + +Their bridge-filtering blocks are deliberately identical. Keep them that way: +that is the capability the two images silently disagreed on. + +A fragment line is a request, not a guarantee. Kconfig silently drops a symbol +whose dependencies are unmet, and silently clamps a tristate to the value of +what it depends on — `CONFIG_BRIDGE_NF_EBTABLES=y` under `CONFIG_BRIDGE=m` +becomes `=m`, and the build still succeeds. Check the produced `.config` rather +than assuming, with `os/common/scripts/check-kernel-config.sh <.config> +`; both backends run it, mkosi during the kernel build and Yocto +in `os/yocto/scripts/export-artifacts.sh` before anything is published. +`os/common/scripts/check-lxc-kernel-config.sh <.config>` runs at the same two +points and is the second gate to satisfy: it fails on anything LXC's +`lxc-checkconfig` reports missing, with the two `IP*_NF_TARGET_MASQUERADE` +compat aliases (pure `select`s of `NETFILTER_XT_TARGET_MASQUERADE`, which both +kernels build) as the only exemptions. + +On the Yocto backend a module also has to be *packaged* into the rootfs. +`RDEPENDS:${KERNEL_PACKAGE_NAME}-base` is cleared in +`layers/meta-dstack/conf/machine/dstack.conf`, so the image contains only the +`kernel-module-*` packages named explicitly in +`layers/meta-dstack/recipes-core/images/dstack-rootfs-base.inc` plus those +pulled in by `RRECOMMENDS` — notably the `ip6table_*` and `xt_checksum` modules +recommended by the `iptables` recipe, which are installed only once the kernel +actually builds them. The mkosi backend runs a full `modules_install`, so every +module its config produces is present. diff --git a/docs/libvirt-network-filter.md b/docs/libvirt-network-filter.md new file mode 100644 index 000000000..1e161c183 --- /dev/null +++ b/docs/libvirt-network-filter.md @@ -0,0 +1,233 @@ +# Optional libvirt network filtering + +## Goal + +Allow bridge-backed VMs to opt into an existing libvirt `nwfilter` without +allowing libvirt to create or launch the QEMU domain. QEMU remains entirely +owned by `dstack-vmm`, so its command line and attestation inputs do not change +outside the explicitly selected network backend. + +This integration applies only to bridge interfaces. It does not filter user, +custom, or macvtap networking. In particular, macvtap bypasses the Linux +bridge and requires policy enforcement in the physical network or a separate +host mechanism. + +The measurable acceptance criteria are: + +- `network_filter = "none"` installs no nwfilter binding. It does not remove + the `netd` dependency: `netd` creates the TAP for every bridge NIC either + way, and the VMM uses `-netdev tap` either way. What changes is only whether + that TAP carries a binding. +- `network_filter = "libvirt"` creates the TAP and filter binding before QEMU + is submitted to Supervisor, and uses QEMU `-netdev tap`. +- A failed TAP or filter setup prevents QEMU from starting and rolls back all + interfaces prepared for that VM. +- Normal stop and removal delete the filter binding and TAP. +- One host `netd` can serve multiple VMM instances. Resource names include a + stable VMM instance namespace, VM ID, and NIC index. +- Development builds can run the VMM and `netd` directly, with explicit socket + and allowed-UID command-line options; systemd is not required. + +## Configuration + +Filtering is a VMM host policy, not a field accepted from a VM manifest: + +```toml +[cvm.network_filter] +mode = "none" # or "libvirt" +filter = "clean-traffic" +parameters = {} + +[netd] +socket = "/run/dstack/netd.sock" +socket_mode = 0o660 # used when netd binds the socket itself +libvirt_uri = "qemu:///system" +``` + +Deployment RPC network choices are separately constrained by node policy: + +```toml +[cvm] +allowed_network_modes = ["user", "bridge"] +allowed_bridges = ["tenant-br0"] +allowed_macvtap_parents = [] +``` + +Macvtap is excluded from `allowed_network_modes` by default. Empty bridge and +macvtap-parent allowlists prevent RPC callers from overriding the respective +node defaults. If macvtap is explicitly enabled, callers may select a +parent listed in `allowed_macvtap_parents`, the node's own configured parent, or +one this VM already holds — restating a value the node would have supplied +anyway grants nothing new. The same applies to `bridge_name` and +`allowed_bridges`. The macvtap forwarding mode always comes +from `[cvm.networking].macvtap_mode` and cannot be selected through deployment +RPCs. These allowlists authorize attachment targets; an nwfilter is not a +substitute for that authorization. + +Each VMM instance also has an `instance_id`. It must be unique among VMMs that +share a host. If omitted, the VMM derives a stable namespace from its absolute +run directory. + +## Architecture + +`netd` is a host-level privilege broker. It serves the `Netd` pRPC service +(`dstack/vmm/netd-rpc/proto/netd_rpc.proto`) over a Unix stream socket, with +one method per operation. The socket's filesystem owner, group, and mode are +the authorization boundary: every process that can connect is fully trusted to +call every netd method. A single process can serve multiple VMMs; a dedicated +process can use another socket for development or isolation. + +The VMM and netd must come from the same release. After upgrading, restart +`dstack-netd` together with the VMM; a VMM cannot talk to a netd from a release +that predates the pRPC service. + +When netd creates the socket itself, `socket_mode` defaults to `0o660`. Ensure +that each authorized VMM process can reach the socket through its owning group +or another deliberately configured ownership arrangement. Never make the +socket accessible to an untrusted local user. + +For libvirt mode, startup is: + +1. Derive the TAP name from instance namespace, VM ID, and NIC index. +2. Create the TAP for the configured QEMU UID and attach it to the bridge. +3. Create a libvirt nwfilter binding for the TAP. +4. Bring the TAP up and return success. +5. Start QEMU directly with `-netdev tap,script=no,downscript=no`, carrying + `vhost=on|off` and, above one queue pair, `queues=N`. + +Teardown stops QEMU first, removes the binding, and deletes the TAP. `netd` +runs operations one at a time, in the order they arrive. The design intentionally does not add ownership +aliases; deployments must use unique instance IDs. + +The protocol does not pin a client to an instance namespace, so any process +with socket access can prepare, check, or remove any deterministic identity. +Deploy VMM instances that do not share this trust boundary with dedicated netd +sockets and distinct filesystem permissions. + +`netd` invokes fixed absolute `ip` and `virsh` executables with separate +arguments. It never accepts a command, executable path, TAP name, or raw XML +from a client. Filter XML is generated internally with XML escaping and is +validated by libvirt. + +## Deployment modes + +Production should run one shared service. `netd` reads the `[netd]` section, +plus `cvm.network_filter.mode` if the file has one, so its root-owned +configuration can be small and independent of every VMM instance: + +```toml +# /etc/dstack/netd.toml +[netd] +socket = "/run/dstack/netd.sock" +socket_mode = 0o660 +libvirt_uri = "qemu:///system" + +# Required here because this file has no [cvm] section for netd to read the +# node's policy from. +[netd.network_filter] +mode = "libvirt" +filter = "clean-traffic" +parameters = {} +``` + +`[netd.network_filter]` is netd's own copy of the invariant, not a convenience. +netd is the privileged side of the socket, and anything that can reach the +socket can ask for an unfiltered TAP on a host bridge — a request a filtering +node has to refuse in the daemon rather than in its caller. When netd and the +VMM share one `vmm.toml`, leaving it unset derives it from +`[cvm.network_filter]` so the two cannot drift apart; a malformed section is a +startup error rather than a silent fallback to "filter nothing". + +The request says only *whether* to bind a filter, never which one. A caller that +named the filter could name `allow-arp`, which contains no drop rule at all, or +pin `clean-traffic` to the gateway's MAC and IP through its parameters, and +still satisfy a policy that asked for "some filter". + +A macvtap parent is refused when filtering is required and the parent is a host +bridge or is enslaved to one: nwfilter does not apply to macvtap, so that +request is the same unfiltered access to the same segment, spelled with a +different operation. + +Production deployments can use systemd socket activation. The socket unit +owns the filesystem mode and ownership; `netd.socket_mode` applies only to the +standalone bind path. + +```ini +# /etc/systemd/system/dstack-netd.socket +[Unit] +Description=dstack host networking socket + +[Socket] +ListenStream=/run/dstack/netd.sock +SocketMode=0660 +SocketUser=root +SocketGroup=dstack-vmm +RemoveOnStop=true + +[Install] +WantedBy=sockets.target +``` + +```ini +# /etc/systemd/system/dstack-netd.service +[Unit] +Description=dstack host networking service +After=libvirtd.service + +[Service] +ExecStart=/usr/bin/dstack-vmm --config /etc/dstack/netd.toml netd +Restart=on-failure +``` + +The service accepts exactly one Unix stream listener through the systemd +`LISTEN_FDS` protocol. With no activated descriptor it falls back to binding +`netd.socket` itself. More than one descriptor, or a descriptor of the wrong +socket type, is rejected. + +All VMM instance configurations point to the same socket and use distinct +`cvm.instance_id` values. A dedicated netd uses a different socket. A +host-wide lock serializes mutations made by shared and dedicated netd +processes. + +Development mode is two ordinary commands: + +```bash +sudo dstack-vmm --config ./vmm.toml netd \ + --socket /run/dstack-dev/netd.sock +sudo dstack-vmm --config ./vmm.toml \ + --netd-socket /run/dstack-dev/netd.sock +``` + +User networking and a caller-supplied netdev never ask `netd` to build an +interface. Bridge and macvtap do ask, and fail closed if `netd` is +unavailable. + +Filtered TAP netdevs follow the node's `vhost` and `queues` settings like any +other TAP-backed NIC (see [network-data-plane.md](network-data-plane.md)). The +nwfilter binding is installed on the host TAP interface, so packets traverse it +whether they were written by QEMU or by a vhost worker; filtering is unaffected +by the data plane choice. Enabling vhost does require the QEMU user to be able +to open `/dev/vhost-net`. + +`netd` creates the TAP for unfiltered bridge NICs too. Those TAPs carry no +nwfilter binding, so a bridge node needs `netd` even when +`network_filter.mode = "none"` — see +[bridge-networking.md](bridge-networking.md) for why the host interface has a +single owner. + +An empty filter name is what selects that unfiltered TAP, so `mode = "libvirt"` +with an empty `filter` is rejected at config load rather than quietly producing +an unbound TAP. + +Removal carries the same distinction: the VMM tells `netd` whether the interface +it is asking about was created with a binding, from a record made when it was +built rather than from configuration that may have changed since. A binding it +was told about must be gone before `netd` returns; otherwise `netd` still asks +libvirt to clear one — an interface name is reused by the same VM, and a +leftover binding's rules would be inherited — but a `libvirtd` it cannot reach +is a warning rather than a failure. So a node with `virsh` installed and no +running `libvirtd` can create and destroy multiqueue TAPs. The flag defaults to +true on the wire, so an older VMM's removals still drop their bindings. + +`netd` requires the `virsh` binary to be present whatever the filter mode; it is +`libvirtd` that unfiltered work does not need. diff --git a/docs/macvtap-networking.md b/docs/macvtap-networking.md new file mode 100644 index 000000000..362de4e87 --- /dev/null +++ b/docs/macvtap-networking.md @@ -0,0 +1,76 @@ +# Macvtap networking + +Macvtap mode gives each CVM a layer-2 identity on an existing host network +without adding the parent interface to a Linux bridge. The VMM delegates the +privileged interface lifecycle to `dstack-vmm netd`; manifests never contain +the unstable `/dev/tapN` device path. + +## Configuration + +Configure a NIC through node configuration or an authorized VMM RPC request: + +```json +{ + "mode": "macvtap", + "parent": "eth0", + "macvtap_mode": "private" +} +``` + +`parent` must name an existing host interface. `macvtap_mode` may be +`private`, `bridge`, `vepa`, or `passthru`; an empty value selects `private`. +Macvtap NICs also honour the `vhost` and `queues` settings described in +[network-data-plane.md](network-data-plane.md); netd creates the interface with +matching hardware queues and the launcher opens `/dev/tapN` once per queue. +The configured netd socket permissions apply in the same way as for +libvirt-filtered bridge networking. + +Deployment RPC callers cannot select `macvtap_mode`; it is inherited from the +node's `[cvm.networking]` configuration. Macvtap is also excluded from the +default RPC policy. A node operator must explicitly enable it and enumerate +the host interfaces callers may select: + +```toml +[cvm] +allowed_network_modes = ["user", "bridge", "macvtap"] +allowed_macvtap_parents = ["eth0"] + +[cvm.networking] +mode = "user" +parent = "eth0" +macvtap_mode = "private" +``` + +## Lifecycle + +For every macvtap NIC, the VMM sends netd the VM identity, NIC index, parent, +and the same deterministic MAC address passed to QEMU. Netd then: + +1. derives the stable `dt` interface name; +2. replaces any stale interface with that name; +3. creates and activates the macvtap interface; +4. reads its kernel-assigned ifindex and waits for `/dev/tap`; and +5. returns that runtime device path to the VMM. + +The per-VM launcher opens the character device once per queue pair, places the +descriptors at the fds referenced by QEMU's `-netdev tap,fd=...` (or `fds=...`) +argument, and then execs QEMU. This keeps device +paths out of persistent VM +configuration, works with both Supervisor and systemd process managers, and +does not pass network fds through `sudo`. + +VM shutdown removes the interface by its deterministic identity. The device +node disappears with the interface; its numeric path is never reused as an +identity or cleanup key. + +## Limitations + +- The host and a macvtap guest do not communicate directly through the parent + interface by default. Add a host macvlan/macvtap endpoint if that path is + required. +- Libvirt nwfilter bindings apply only to bridge mode and are never installed + for macvtap interfaces. Macvtap deployments + must enforce network policy in the physical network or with another host + mechanism. +- Real-host testing requires `CAP_NET_ADMIN`, a working udev setup for + `/dev/tapN`, and an upstream network that accepts multiple MAC addresses. diff --git a/docs/nerdctl-compose.md b/docs/nerdctl-compose.md new file mode 100644 index 000000000..83ecf256b --- /dev/null +++ b/docs/nerdctl-compose.md @@ -0,0 +1,63 @@ +# nerdctl Compose and lazy image pulling + +dstack supports two independent Compose runners. The runner is part of +`app-compose.json`, so it is included in the compose hash and attestation. + +| `runner` | Image manager | `snapshotter` support | +|---|---|---| +| `docker-compose` | Docker Engine | none (Docker's overlayfs store) | +| `nerdctl-compose` | containerd | `overlayfs` or `stargz` | + +## Deploy with stargz + +Build and push an eStargz image before deployment. For example, with Buildx: + +```bash +docker buildx build -t registry.example.com/example/app:estargz \ + --output type=registry,oci-mediatypes=true,compression=estargz,force-compression=true \ + . +``` + +Reference that image from `docker-compose.yaml`, then deploy it with: + +```bash +dstack deploy -c docker-compose.yaml \ + --runner nerdctl-compose \ + --snapshotter stargz +``` + +The resulting application manifest contains: + +```json +{ + "manifest_version": "3", + "runner": "nerdctl-compose", + "snapshotter": "stargz" +} +``` + +Use containerd without lazy pulling by selecting overlayfs: + +```bash +dstack deploy -c docker-compose.yaml \ + --runner nerdctl-compose \ + --snapshotter overlayfs +``` + +If `snapshotter` is omitted for `nerdctl-compose`, it defaults to `overlayfs`. +Setting `snapshotter` with `docker-compose` is rejected rather than silently +ignored. + +## Compatibility + +`nerdctl compose` implements the commonly used Docker Compose features, but it +is not a drop-in implementation of every Docker-specific extension. Test +applications that use Docker socket mounts, custom runtimes, or advanced +networking before switching runners. The `nerdctl-compose` runner requires +pre-built images and rejects Compose `build` sections. This avoids depending +on an in-guest BuildKit daemon and ensures lazy-pull images were converted +before deployment. + +Both backends keep their own image and container metadata. Changing the runner +recreates the application through the selected backend; it does not migrate +existing Docker containers into containerd. diff --git a/docs/network-data-plane.md b/docs/network-data-plane.md new file mode 100644 index 000000000..c80af880f --- /dev/null +++ b/docs/network-data-plane.md @@ -0,0 +1,293 @@ +# virtio-net data plane tuning + +Every CVM NIC has two knobs that decide how many packets it can move: whether +the host kernel's vhost-net data plane is used, and how many virtio-net queue +pairs the device exposes. vhost is set per node and overridable per VM; queue +pairs have no node-wide setting at all, for the reason given under +Configuration. + +## Why it matters + +Without vhost-net, QEMU drains every received packet on its single main-loop +thread. That thread is the ceiling, and it does not grow with vCPUs: + +``` +maximum packets per second ≈ 1 core ÷ per-packet main-loop cost +``` + +The per-packet cost varies with traffic shape — a few microseconds for uniform +synthetic streams, tens of microseconds for bidirectional short-connection +traffic — so the ceiling is a property of the workload, not a fixed number. +What is fixed is the shape of the failure: throughput climbs normally until the +main loop saturates at 100% of one core, then packets are dropped at the TAP +before they ever reach the guest. Guest-side counters stay clean, which makes +the cliff easy to misdiagnose as a network problem. + +Guest-side outbound traffic uses the same thread, so a busy guest pays the +cost twice over. + +`vhost=on` moves that work into the host kernel. That returns a whole core, but +it relocates the ceiling rather than removing +it: packets now arrive faster than a single guest receive queue can drain, and +the drops reappear at a higher rate. More queue pairs is what removes them, +which is why enabling vhost also enables a queue count that follows the VM's +vCPU count — the two travel together. + +vhost is **off by default** and enabled per node (or per VM). Two things make +it an opt-in rather than a default: turning it on changes the virtio-net device +of every bridge/macvtap VM on its next boot, and it requires `/dev/vhost-net` +to be accessible to the account QEMU runs under, which the VMM cannot verify on +the operator's behalf — see [Enabling vhost on a node](#enabling-vhost-on-a-node). + +## Configuration + +```toml +[cvm] +# Ceiling for both the default and what a deployment may request. +max_net_queues = 16 + +[cvm.networking] +mode = "bridge" +bridge = "dstack-br0" +vhost = true +``` + +Queue pairs are not a node setting. With vhost on they default to the VM's vCPU +count, capped at 16, because the useful number follows the VM rather than the +host — the guest driver uses at most one queue pair per vCPU. A deployment +overrides that per VM, up to `max_net_queues`. + +Raising `max_net_queues` above 16 widens what a deployment may ask for without +moving the default's cap, so a larger VM never silently acquires a worse +default. Lowering it below 16 does lower the default too, because a node that +refuses a request for four queue pairs should not hand out sixteen by itself. +The hard ceiling from any source is 64. + +Without vhost the default is a single queue pair. The QEMU main loop drains +every queue on one thread, so extra queues buy little while still costing more +MSI-X vectors and a changed guest device. An explicit +queue count is still honoured without vhost, since that combination is a +deliberate request rather than a default. The two defaults travelling together +also means a node that never sets `vhost` keeps building the device its VMs +have always had. + +A VM overrides either value at deploy time, and `UpdateVm` changes them +afterwards — the new values apply from the VM's next boot: + +```bash +vmm-cli.py deploy --name my-vm --image dstack-0.5.9 --compose app.yaml \ + --net bridge --net-queues 4 +vmm-cli.py deploy --name latency-vm --image dstack-0.5.9 --compose app.yaml \ + --net bridge --net-no-vhost + +# retune an existing VM +vmm-cli.py update --net-queues 8 --net-vhost + +# stop pinning either value and follow the node and the vCPU count again +vmm-cli.py update --net-queues auto +vmm-cli.py update --net-vhost-default + +# stop pinning the backend too, and follow whatever the node runs +vmm-cli.py update --net default +``` + +Every pin has an un-pin. A VM keeps reporting whatever it pinned, and the +deployment RPC accepts a VM's own held values back even after the node's +policy moves, so a read-modify-write update never strands a VM. + +The web UI exposes both per NIC in the deploy and update dialogs, alongside the +networking mode. Both fields are also on `NetworkingConfig` in the deployment +and update RPCs. A request that +sets only `vhost`/`queues` keeps the node's own networking mode, so tuning does +not force a caller to restate — or be allowed to choose — a backend. `queues` is +rejected above the node's `max_net_queues`; `vhost` is not otherwise restricted, +since it only affects the requesting VM. `GetMeta` reports +`networking.max_queues` so a client can present the real bound. + +The data plane settings are recorded only when a deployment asks for them. +Leave one out and it stays owned by the node, so changing `[cvm.networking]` +later — including setting `vhost = false` to roll the whole node back — still +reaches VMs deployed with some other networking override. + +Naming a backend is different: it pins that NIC's *backend*, resolved at +deployment. Its mode, and the bridge or macvtap parent that names it, are fixed +for the life of the VM, so a later edit to those fields in `[cvm.networking]` +does not move it to another segment. Nothing else is pinned — the MAC prefix, +the user-mode subnet and DHCP start, and the macvtap forwarding mode stay node +settings and are re-read at every launch, so editing them changes every VM's +next boot, including its MAC and therefore its DHCP lease. A request that only +tunes pins nothing at all, including the backend it inherited. + +`GetInfo` reports that configuration back, and both `vmm-cli.py update` and the +web UI read it, change one field, and resend the rest. Two things follow. A +request may name a bridge or macvtap parent the node itself configured even when +the allowlists are empty: leaving the field out already yields exactly that +value, so echoing it grants nothing policy was withholding. And an update may +restate whatever its own VM already pinned, so that moving the node's default +out from under a VM does not leave that VM's configuration unsendable. A NIC +that inherited its backend reports an empty mode, which is the same thing it was +deployed with. + +Neither field reaches the CVM's measurement. The measured VM configuration the +VMM controls covers the OS image, the vCPU and memory counts, several QEMU +layout flags, the *number* of NICs, and `mr_config_id`; the queue count and the +vhost state are not part of it, so retuning a NIC does not change app identity +or require an on-chain update. Adding or removing a NIC does: the NIC count +changes the guest's ACPI tables and therefore RTMR0. + +## Enabling vhost on a node + +Setting `vhost = true` in `[cvm.networking]` is a node-wide behaviour change: +every bridge or macvtap VM that has not pinned its own data plane gets a +different virtio-net device on its next boot — `vhost=on`, `mq=on` with +vCPU-scaled queue pairs, and the matching MSI-X vector count. The device is not +measured, so attestation and app identity are unaffected. Before flipping it: + +1. **Verify `/dev/vhost-net` is accessible to the account QEMU runs under.** + It is `root:kvm 0660` on Debian-family hosts, where adding the account to + the `kvm` group suffices, and `root:root 0600` on several others. If the + account lacks access, QEMU exits at launch and every affected VM stops + restarting. The VMM warns at startup when its own access fails, but it + cannot refuse on that basis — QEMU need not share its credentials. + +2. **Restart `netd` before or together with the VMM.** Every bridge and + macvtap NIC is prepared by `netd`, and the VMM checks that `netd` echoes the + queue count it built. An older `netd` fails that check; the launch is rolled back + and fails with the reason in the VMM log, but the VM does not start until + `netd` is upgraded. + +3. **Roll back by setting `vhost = false`.** The node value reaches every VM + that did not pin `vhost` explicitly, from its next boot; a VM that pinned + `vhost = true` keeps it until updated. + +## What each mode supports + +| Mode | netdev | vhost | queues > 1 | +|---|---|---|---| +| `user` | `user,...` | no backend | not supported | +| `bridge` | `tap,ifname=` via netd | yes | yes | +| `macvtap` | `tap,fd=` / `tap,fds=` | yes | yes | +| `custom` | operator's own string | operator's own string | no, not settable | + +QEMU's `bridge` netdev accepts neither `vhost=` nor `queues=`, and the setuid +`qemu-bridge-helper` behind its `tap` netdev returns exactly one descriptor. So +bridge mode runs on a TAP that `netd` creates: persistent, `multi_queue` when +asked for, and opened once per queue by QEMU. That is true of every bridge NIC, +filtered or not, single-queue or not — see +[bridge-networking.md](bridge-networking.md) for why the host interface has one +owner. `netd` requires the `virsh` binary to be installed even when nothing is +filtered, though it does not require a reachable `libvirtd`. One-shot +`dstack-vmm run` does not manage netd interface lifecycle, so it refuses bridge +and macvtap NICs outside `--dry-run`. `netd` reports back the queue count it +created, and the VMM refuses to launch on a mismatch — a `netd` deployed +separately as a root service can be older than the VMM asking it for +multiqueue, and QEMU would otherwise reject the interface from inside the +per-VM launcher. + +For macvtap, the per-VM launcher opens the `/dev/tapN` character device once +per queue pair and hands QEMU the descriptors as `fds=`. `netd` creates the +interface with matching `numtxqueues`/`numrxqueues`. + +Custom mode owns its whole netdev string, including any `vhost=`/`queues=` +options, and its guest device stays single-queue: the VMM cannot edit that +string, so it has no way to make a multiqueue device line agree with it. For the +same reason `GetInfo` reports no vhost state and no queue count for a custom +NIC, rather than asserting the resolved defaults over a string it never read. A +hand-written multiqueue netdev will not pair with a multiqueue guest device +today. + +Naming a backend that cannot carry vhost or a queue count, and then asking for +one, is refused — the request is yours to correct. Inheriting such a backend is +not, because the node chose it and may choose another tomorrow; the request +reads as off, or as one queue pair, until then. + +## Choosing a queue count + +The default suits bandwidth-bound workloads. Latency-sensitive ones should ask +for fewer: more queues spread receive processing over more vCPUs, and under TDX +a cross-vCPU wakeup costs an IPI and a VM exit. Measured on one 8-vCPU TDX CVM, +changing only the guest's channel count: + +| Queue pairs | Short-connection throughput | +|---|---| +| 1 | 22.3k conn/s | +| 2 | ~20k conn/s | +| 4 | 15–21k conn/s | +| 8 | 6.2–7.7k conn/s | + +The same CVM with 8 queues moved 3.0 Mpps of 64-byte UDP with no loss, against +roughly 600k with one queue. The trade is real in both directions, so a VM +serving many short connections should set `--net-queues 1` and measure. + +A VM with fewer vCPUs than queues leaves the extra pairs idle — `ethtool -l +eth0` reports the smaller number. An explicit over-provision is not rejected at +deployment, because `vmm-cli.py resize` can raise the vCPU count later. + +Queue pairs also cost guest memory — each RX ring keeps 256 page-sized buffers +posted, about 1 MB per queue pair plus per-queue NAPI and socket state — but at +any realistic RAM/vCPU shape this is noise. Pushed to a shape no deployment +uses (1 GB of RAM with 16 vCPUs, so 16 queue pairs by default), sustained load +did produce atomic order-0 page-allocation failures in RX refill +(`try_fill_recv` in the guest log); 2 GB at the same shape ran clean. Since the +queue default follows the vCPU count and a VM with that many vCPUs carries far +more memory in practice, this needs no tuning — it is recorded here so the +symptom is searchable. The TDX bounce-buffer pool is not a constraint either: +the guest kernel sizes swiotlb at 6% of RAM clamped to [64 MB, 1 GB] with no +`swiotlb=` parameter, while peak demand is bounded by ring size at about 2 MB +per queue pair — a deliberately undersized 32 MB pool sustained full +multiqueue line rate with zero `swiotlb buffer is full` events. + +`vectors` is derived, never configured: `2N + 2`, one vector per queue +direction plus config and control. One queue pair emits no `mq=on` or +`vectors=` at all, leaving the guest device line byte for byte identical to the +one before this feature. The `-netdev` half carries `vhost=on|off` either way, +since that is what selects the backend. + +## Requirements + +The account running QEMU must be able to open `/dev/vhost-net`, which is +`root:kvm 0660` on a stock host — add that account to the `kvm` group. The +`vhost_net` module autoloads on first open. + +`GetInfo` reports the data plane each interface actually got, so a bridge NIC +that fell back for want of a helper reads as `vhost: false` rather than +advertising something it is not using. For a VM that is not running there is no +interface to describe, so it reports what the next launch would build instead -- +the same calculation, against the node configuration and manifest as they stand +now, rather than the ones a finished boot ran under. + +If that account lacks access, QEMU exits at startup and the VM never boots — +there is no fallback to the userspace backend at this point, on any QEMU +version (verified on 8.2.2 and 10.2). What the per-VM launcher log shows +depends on the version: QEMU 8.2 prints `warning: tap: open vhost char device +failed: Permission denied` (once per queue) and then dies on `net/net.c:1185: +net_client_init1: Assertion 'nc' failed` — an upstream bug +([qemu#1486](https://gitlab.com/qemu-project/qemu/-/issues/1486)); later +versions exit cleanly with `Could not open '/dev/vhost-net'`. Grep for either. +The VMM does not refuse a launch over this: QEMU need not share the VMM's +credentials, so a refusal based on the VMM's own access would block deployments +the host can run. It warns instead — when the device node is missing outright, +and when the VMM's own open is denied, since QEMU usually does share its +account. + +QEMU does have a *runtime* fallback, at a different failure point: once the +netdev initialized with vhost, a later `vhost_net_start()` failure at guest +driver activation logs `unable to start vhost net: : falling back on +userspace virtio` and keeps the NIC working on the userspace data path. That +path is reachable only after `/dev/vhost-net` was opened successfully at +launch, so an access problem never lands there. If it does fire, it is the one +case where `GetInfo` can overstate the data plane — the interface reports the +vhost state the launch settled while the packets take the userspace path — and +that QEMU log line is the indicator. + +vhost-net works normally in a TDX guest: the virtio rings and buffers live in +shared, unencrypted memory precisely so a host-side backend can reach them. +This is the same mechanism behind `vhost-vsock-pci`, which dstack has always +used. + +On host kernels older than 6.4 the vhost worker is a free-standing kernel +thread: it is attached to the owner's cgroups, so `cpu.max` and cgroup +accounting do apply, but it is outside QEMU's thread group and so invisible to +`top -H` and to anything reading `/proc//task`. Since 6.4 it is a +`vhost_task` inside that thread group and shows up everywhere the VM's other +threads do. diff --git a/docs/onboarding.md b/docs/onboarding.md index e36d89f71..28b216ff2 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -3,7 +3,7 @@ Use this guide to get a first dstack app running on one Intel TDX host. The workflow uses `dstackup` for host setup and `dstack` for app deployment: ```bash -curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/dstack/scripts/install.sh | sh +curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/next/dstack/scripts/install.sh | sh sudo dstackup install sudo dstack deploy \ -n hello-nginx \ @@ -78,7 +78,7 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y Build and install the `dstackup` bootstrap command: ```bash -curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/dstack/scripts/install.sh | sh +curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/next/dstack/scripts/install.sh | sh ``` The bootstrap installer builds `dstackup` from a temporary source checkout and installs it under `/usr/local/bin`. The `dstackup install` command then builds and installs `dstack`, `dstack-auth`, `dstack-vmm`, `supervisor`, static assets, and host config into the system layout. @@ -149,7 +149,7 @@ The deploy command: Pass `--vcpu`, `--memory`, or `--disk` to change the app resources before you deploy. -The `--port 8080:80` mapping means `host_port:vm_port` and uses TCP on `127.0.0.1`. The full accepted forms are `vm`, `host:vm`, `proto:host:vm`, and `proto:addr:host:vm`. Use `tcp` or `udp` for `proto`. Fixed host and VM ports must be between 1 and 65535. If you omit the host port, or use `auto` or `0`, `dstack` picks a free localhost port and prints the selected mapping after deploy. +The `--port 8080:80` mapping means `host_port:vm_port` and uses TCP on `127.0.0.1`. The full accepted forms are `vm`, `host:vm`, `proto:host:vm`, and `proto:addr:host:vm`, each optionally suffixed with `@` to name which NIC the traffic enters through (a single-NIC VM never needs it). Use `tcp` or `udp` for `proto`. Fixed host and VM ports must be between 1 and 65535. If you omit the host port, or use `auto` or `0`, `dstack` picks a free localhost port and prints the selected mapping after deploy. Open the app from the host: @@ -216,7 +216,7 @@ Use `--prefix` when you want a second isolated install on the same host. A custo Install `dstackup` into the prefix, then use the same prefix for `dstackup` and `dstack`: ```bash -curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/dstack/scripts/install.sh | sh -s -- --prefix /opt/dstack-test +curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/next/dstack/scripts/install.sh | sh -s -- --prefix /opt/dstack-test sudo /opt/dstack-test/bin/dstackup install \ --prefix /opt/dstack-test \ diff --git a/docs/rc-testing-runbook.md b/docs/rc-testing-runbook.md new file mode 100644 index 000000000..1e4478679 --- /dev/null +++ b/docs/rc-testing-runbook.md @@ -0,0 +1,596 @@ +# Release-candidate testing runbook + +What to do, in order, to exercise a dstack release candidate on real TEE +hardware — and the specific things that cost time the last round, so they cost +nothing the next one. + +Written after testing 0.6.0-rc0 on an Intel TDX host and an AMD SEV-SNP host. +Assumes fresh machines: nothing here depends on state left behind by that run. + +--- + +## 1. Confirm the host can actually do what you think + +Get this wrong and you will scope the whole test around a capability you have, +or skip one you don't. + +### Intel TDX host + +```bash +cat /sys/module/kvm_intel/parameters/tdx # expect: Y +grep -o -m1 tdx_host_platform /proc/cpuinfo # expect: tdx_host_platform +systemctl is-active qgsd pccs # expect: active active +``` + +**`/dev/tdx_guest` does not exist on a TDX host and its absence means nothing.** +That device is the *guest*-side attestation interface; it appears inside a TD, +not on the machine running them. Checking for it will tell you a perfectly good +TDX host has no TDX. Use the KVM module parameter and the CPU flag. + +`qgsd` is what turns a TD's attestation request into a signed quote, and `pccs` +caches the Intel collateral used to verify it. Without both, CVMs boot but +cannot obtain keys. + +### AMD SEV-SNP host + +```bash +cat /sys/module/kvm_amd/parameters/sev_snp # expect: Y +grep -o -m1 sev_snp /proc/cpuinfo # expect: sev_snp +ls -l /dev/sev # must exist AND be accessible +qemu-system-x86_64 -object help | grep sev-snp-guest +lsmod | grep -E '^vhost_vsock|vmw_vsock_vmci' +``` + +Three host-level problems each blocked us for a while: + +- **`vhost_vsock` not loaded.** The VMM binds its host API on vsock CID 2 and + fails with `Cannot assign requested address (os error 99)`. Fix: `sudo modprobe + vhost_vsock`. Does not survive reboot — add `/etc/modules-load.d/` to persist. + +- **VMware's vsock transport hijacking the stack.** If `vmw_vsock_vmci_transport` + is loaded (it often is, by default), it claims the vsock transport and CID 2 + binding still fails even with `vhost_vsock` present. Fix: `sudo modprobe -r + vmw_vsock_vmci_transport vmw_vmci`. It comes back on reboot — blacklist to + persist. This one is easy to misdiagnose as a dstack bug. + +- **`/dev/sev` is `root:root 0600` out of the box** and your user is probably not + in `kvm`. QEMU then fails with `Could not access KVM kernel module: Permission + denied`. Fix: `sudo usermod -aG kvm $USER` and `sudo chown root:kvm /dev/sev && + sudo chmod 660 /dev/sev`. The chmod does not survive reboot; a udev rule does. + +**After fixing group membership, restart the supervisor.** `dstack-vmm` starts +`supervisor` detached, and a supervisor started before the `usermod` keeps the old +group set. QEMU is forked from it, so it still cannot open `/dev/kvm` even though +your shell can. Symptom: `id` shows `kvm`, QEMU still says permission denied. +Check with `grep ^Groups /proc/$(pgrep -f bin/supervisor)/status`. + +### QEMU version + +dstack computes measurements against an ACPI compatibility profile keyed on the +QEMU major version (`dstack/crates/qemu-acpi/src/profile.rs`). 8.x, 9.x and 10.x +are all mapped. Check the version resolves to a supported profile *before* +installing anything, or you will build an environment that cannot produce correct +measurements. + +--- + +## 2. Pick your disks deliberately + +Check the mount that will actually hold the data, not `/`: + +```bash +df -h /home ~/.dstack /var/lib/docker +``` + +On one host `/` had 12 GB free while `/home` sat on a 7 TB array. Looking only at +`/` produced a wrong "we are too tight on disk to use the documented `--disk 50G`" +conclusion and a plan built around shrinking things unnecessarily. + +Keep the VMM's `run_path`, `temp_dir` and the image directory on the large +volume. CVM disks and temp files land there, and `/tmp` on a small root +filesystem will fill up mid-test. + +--- + +## 3. Two traps that can reach outside the test + +**`~/.dstack-vmm/config.json` may point `vmm-cli.py` at a production VMM.** It is +the CLI's default `--url`. On both machines used last time it pointed at a live +deployment with credentials, so any `deploy` that omitted `--url` would have +landed there. **Pass `--url http://127.0.0.1:` on every single invocation.** + +**Component tags publish to Docker Hub.** `kms-v*`, `gateway-v*`, `verifier-v*` +and friends trigger release workflows that push to `${DOCKERHUB_ORG}` and cut +GitHub releases. If you only want images in a private registry, build locally and +push by hand — do not push those tags. + +--- + +## 4. Cutting the release + +- Version lives in **four** places with no tool keeping them in sync: + `dstack/Cargo.toml` (`workspace.package.version`), `os/mkosi/versions.env` + (`DSTACK_VERSION`), `os/mkosi/mkosi.conf` (`ImageVersion`), and + `os/yocto/.../dstack.conf` (`DISTRO_VERSION`). Regenerate `Cargo.lock` after. + Only the first two are cross-checked (by `os/mkosi/tests/acceptance.sh`); the + Yocto one is checked by nothing. +- **Commit messages must be conventional commits** — `prek.toml` enforces it at + commit-msg stage. The historical `Bump version to X` style now fails. +- `release/**` branches are protected against **deletion and non-fast-forward**. + Rebase-and-force will be rejected. Merge instead. This also means the branch + name is claimed permanently once pushed. +- **The container Dockerfiles clone from GitHub and check out `DSTACK_REV`** — + they do not use your working tree. Push the commit before building images, or + you will build something other than what you are testing. +- Pre-run the release job's own validation locally before tagging: + + ```bash + TAG=mkosi-os-v + echo "$TAG" | grep -Eq '^mkosi-os-v[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.-]+)?$' + source os/mkosi/versions.env && [ "${TAG#mkosi-os-v}" = "$DSTACK_VERSION" ] + ``` + + The release job also requires `metadata.json`'s `git_revision` to equal the + tagged commit and rejects a `-modified` suffix, so the tag must sit on the exact + commit that was built from a clean tree. + +--- + +## 5. Bring-up order + +KMS first, then gateway, then the app. Each needs the previous one's identity. + +1. **VMM.** Copy `dstack/vmm/vmm.toml` and edit: `address` **and** `port` (the + `check-config` subcommand requires both at top level), `run_path`, `temp_dir`, + `[image] path` (it is commented out by default), `platform = "tdx"` or + `"amd-sev-snp"`, a `cid_start` that avoids running VMs, `[supervisor]` paths, + and `[cvm.port_mapping]` — **enabled, with a range covering every port you + intend to map.** A port outside the range fails the deploy with `Port mapping + is not allowed for tcp:`. Validate with `dstack-vmm -c + check-config` before starting. + +2. **KMS as a CVM** with `--local-key-provider` (needs a + gramine-sealing-key-provider on the host, default `127.0.0.1:3443`). The + simplest authorization that releases keys without a chain is + `[core.auth_api] type = "dev"` with `gateway_app_id = "any"` — no extra + containers, unlike the on-chain compose which needs a Helios light client. + Keep `[core.image] verify = true`. + +3. **Gateway as a CVM** with `--kms`. Internal ports are RPC 8000, admin 8001, + WireGuard 51820/udp, proxy 443. + +4. **App CVM** with `--kms --gateway`. + +### Certificate identity across hosts + +If CVMs on more than one machine will talk to the KMS, give it a DNS name rather +than an IP. The RPC certificate's only SAN is `auto_bootstrap_domain`, and a +local CVM dialing `10.0.2.2` while a remote one dials the public IP cannot both +match. A wildcard A record covering `kms.` and `gw.` solves it in +one step. + +### ACME is configured at runtime, not in a file + +In 0.6.0 certbot config lives in the gateway's KvStore, reached through the admin +API — `Admin.SetCertbotConfig`, `Admin.CreateDnsCredential`, +`Admin.AddZtDomain`. The `CF_API_TOKEN` and `ACME_STAGING` variables in +`gateway/dstack-app/deploy-to-vmm.sh` are **vestigial and unused**; setting them +does nothing. + +**Set `acme_url` to staging explicitly.** The default is Let's Encrypt +*production*, and the rate limits there are unforgiving of a test loop: + +``` +https://acme-staging-v02.api.letsencrypt.org/directory +``` + +`AddZtDomain` kicks off issuance asynchronously and takes a lock. A subsequent +explicit `RenewZtDomainCert` returns `{"renewed":false}` immediately while that +is in flight — that is the lock, not a failure. Watch the gateway container log +instead of the return value. + +--- + +## 6. Gateway cluster + +Worth doing: it exercises WaveKV replication, which a single node never touches +even though sync is running. + +**Sync turns itself on when `NODE_ID > 0`** (`entrypoint.sh`: +`SYNC_ENABLED=$([ "$NODE_ID" -gt 0 ] && ...)`). A single node with `NODE_ID=1` is +therefore already running WaveKV as local storage — you will see `WaveKV: +detected certificate changes` in its log — but nothing has ever replicated. Do +not read those lines as evidence that clustering works. + +### Every node must deploy under the same `--name` + +Cluster members authenticate each other over RA-TLS and **require the peer's +`app_id` to equal their own** (`gateway/src/kv/https_client.rs`). That is +deliberate: it keeps WaveKV replication inside instances of one authorized +compose rather than letting any CVM join. + +`app_id` is the compose hash, and **`name` is part of it**. Deploying a second +node under a different name produces: + +``` +bootnode discovery retry failed: failed to fetch peers from bootnode + app_id mismatch: expected c90cc75a6ceb…, got aa04b7bd7875… +``` + +which looks like a networking or trust problem and is neither. Per-node values — +`NODE_ID`, `WG_IP`, `WG_RESERVED_NET`, `WG_CLIENT_RANGE`, `WG_ENDPOINT`, +`MY_URL`, `BOOTNODE_URL`, `ADMIN_API_TOKEN` — belong in `--env-file`, whose +**values are encrypted at deploy time and are not part of the compose hash**; +only the key names are recorded, as `allowed_envs`. + +So: same `--docker-compose`, same `--name`, different `--env-file`. Confirm +before deploying — + +```bash +sha256sum node1-app-compose.json node2-app-compose.json # must match +``` + +### Allocate per-node resources + +WireGuard subnets follow `SUBNET_INDEX`: node *n* gets `10.8..0/18`, so +index 0 is `10.8.0.0/18` and index 1 is `10.8.64.0/18`. Overlapping them breaks +routing in ways that are tedious to unpick. Each node also needs its own RPC, +admin, proxy and WireGuard ports, and its own `MY_URL` hostname — a wildcard A +record covers `gw.` and `gw2.` without extra DNS work. + +`BOOTNODE_URL` only speeds up discovery; peers are also found through incoming +connections. + +### What proves the cluster actually works + +Peer lists agreeing is the weakest of the three checks. Do all of them: + +```bash +# 1. Both nodes list both peers +curl -H "Authorization: Bearer $TOK" .../prpc/Admin.Status?json | jq '.nodes' + +# 2. The app registered on node1 appears on node2 — registration state replicated +curl -H "Authorization: Bearer $TOK2" .../prpc/Admin.Status?json | jq '.hosts' + +# 3. node2 serves the wildcard certificate it never requested — cert replicated +echo | openssl s_client -connect 127.0.0.1: \ + -servername "-80." | openssl x509 -noout -issuer -subject + +# 4. Traffic through node2 reaches an app whose tunnel terminates on node1 +curl -k --connect-to "-80.::127.0.0.1:" \ + "https://-80.:/" +``` + +The fourth is the one that matters: it shows what replicated is usable routing +state, not just metadata that happens to agree. + +## 7. What to verify, and what each check actually proves + +| Check | Why it is worth doing | +|---|---| +| `sha256sum -c sha256sum.txt`, then `sha256(sha256sum.txt) == digest.txt` | Confirms the artifact matches its own manifest | +| `dstack-mr tdx-measurement-cbor \| cmp - measurement.tdx.cbor` | Confirms the published measurements are *derivable from the image*, not merely asserted | +| Boot all three CVMs to `boot_progress: done` | `done` is only reachable after `GetAppKey` succeeds, so it implies attestation worked | +| `Admin.Status` lists the app with a WireGuard IP | Gateway verified the CVM's RA-TLS quote | +| **A rejected quote in the KMS log** | The one check that proves verification is not a no-op | +| `curl` through the proxy to the app | End to end, but only for one of several proxy modes — see below | +| `/prpc/v1/Info` and `/prpc/Worker.Info` both answer | v1 works and pre-0.6 clients are not broken | +| `dstack-verifier --verify` on a captured attestation, then again on a one-byte-flipped copy | Verification is checked by something other than the KMS, and the crypto is live rather than short-circuited | + +That fifth row is the one people skip. A run where every quote is accepted cannot +distinguish "verification passed" from "verification never ran". Last time the +AMD CVMs supplied it for free — 6 grants and 7 rejections in the same KMS log — +but if everything passes, arrange a failure deliberately. + +The last row is the cheapest way to arrange one. `dstack-verifier --verify +req.json`, with `req.json` holding `{"attestation": ""}` captured from +`/v1/Attest`, gives a full result offline. Flip one byte and re-run: a flip +inside the report signature or the launch measurement must come back +`VEK does not sign the attestation report`. Choose the byte deliberately — +flipping into the outer MessagePack framing instead fails at decode, which proves only +that the parser works. + +### One `curl` does not cover the proxy + +Ingress maps as `[-[][s|g]].`, and the suffixes select +genuinely different code paths: + +| Suffix | Mode | Path | +|---|---|---| +| none | TLS terminated, forwarded as TCP | the common case | +| `s` | TLS passthrough | `proxy/tls_passthough.rs`, SNI-routed, needs a `_dstack-app-address` TXT record | +| `g` | HTTP/2 with TLS termination | gRPC | + +The 0.6.0-rc0 round exercised only the no-suffix path, single-node and then +across a cluster. Passthrough and gRPC were never tried, and neither was anything +beyond a small request — no sustained transfer, long-lived connection or +streaming. If those matter for the release, test them explicitly; a green +no-suffix `curl` says nothing about them. + +Passthrough in particular has its own resolution mechanism: the gateway looks up +`_dstack-app-address.`, falling back to `_dstack-app-address-wildcard.`, +for a TXT record of the form `:`. Seeing a passthrough-resolution +error while testing the *terminated* path usually means the gateway had no +certificate and fell through to SNI routing — fix the certificate, not the DNS. + +**`dstack-mr` binary name collision:** `-p dstack-mr` and `-p dstack-mr-cli` both +produce `target/release/dstack-mr` and overwrite each other. Build only +`-p dstack-mr` for `measure-os` / `inspect-measurement` / `*-measurement-cbor`. + +**Guest API v1 takes bare method names.** `/prpc/v1/Info`, not +`/prpc/v1/Worker.Info`. The v0 and legacy mounts use `trim: "Worker."`, which +*accepts* the prefixed form; v1 has no trim, so the prefix 404s. A 404 there is +your URL, not a broken agent. + +--- + +## 8. Reading DNS evidence without fooling yourself + +Three ways to misread a DNS result, all of which happened: + +- **`dig +short` prints nothing for both NODATA and timeout.** Those are the two + hypotheses you are usually trying to separate. Use full `dig` output and read + `status:` and the `SERVER:` line. +- **certbot deletes the challenge TXT record after a successful issuance.** Its + absence later is cleanup, not a network fault. +- **Cloudflare takes longer than ten seconds to serve a new record from its own + authoritative edge.** An immediate query after creating a record can legitimately + return nothing. + +When testing whether an external service is reachable from an odd vantage point, +**always run the same request against a known-good target through the same path.** +Four public CORS proxies were tried to get an independent view of AMD KDS; every +one returned the identical error for `github.com` as for the target. Without the +control, those errors read as evidence about AMD. + +--- + +## 9. AMD SEV-SNP + +An AMD run was completed end to end: guest SEV-SNP attestation, KMS key release, +independent quote verification, gateway registration and public traffic. Host +prerequisites are in section 1; what follows is everything after the host boots. +None of it is guessable from the code. + +### 9.1 The settings an AMD run needs + +One of these must change or the run cannot succeed; the rest are listed because +their defaults are the ones you want and it is useful to know that before you +start changing things. + +| Setting | File | Default | For an AMD run | +|---|---|---|---| +| `[cvm] platform` | `vmm.toml` (host) | `"auto"` | `"auto"` picks SEV-SNP when the host CPU flags contain `sev_snp`; set `"amd-sev-snp"` to be explicit and to fail loudly on the wrong host | +| `sev_snp_key_release` | `kms.toml`, `[core]` | `false` | **`true`**, or no AMD guest ever gets a key | +| `aws_nitro_tpm_key_release` | `kms.toml`, `[core]` | `false` | leave off; listed because it is the only other gate of this shape | +| `[core.attestation.urls] amd_kds` | `kms.toml`, `dstack-verifier.toml` | `https://kdsintf.amd.com/vcek/v1` | override only to point at a cache or mirror | +| `[core.attestation.root_ca] sev_snp_milan` / `_genoa` / `_turin` | same | unset — vendor ARK compiled in | leave unset unless you are testing against a non-production root | +| `insecure_allow_external_trust_anchors` | same, `[core.attestation]` | `false` | must be `true` if *any* `root_ca` path above is set | + +The KMS side, in full, is two lines: + +```toml +[core] +sev_snp_key_release = true +``` + +Setting a `root_ca` path without the escape hatch is a startup failure, not a +silent fallback: + +``` +external attestation trust anchors are configured but +insecure_allow_external_trust_anchors is false +``` + +The whole `[core.attestation]` block has the same shape in `kms.toml` and +`dstack-verifier.toml`, so a KDS mirror or a test root configured for one can be +copied verbatim into the other. + +### 9.2 What the release gate looks like when it is off + +`sev_snp_key_release` defaults to `false`, and a guest that trips it reboots in a +loop with + +``` +Request failed with status=400 Bad Request, error={ + "error": "amd sev-snp key release is not enabled" +} +``` + +The gate sits *after* full quote verification (`ensure_key_release_allowed`), so +a perfectly good quote still yields nothing — the message is about local policy +and says nothing about the attestation. Only two variants are gated this way: +`dstack-amd-sev-snp` and `dstack-aws-nitro-tpm`. TDX, GCP TDX and Nitro Enclave +have no such switch, which is why nobody notices the gate until the first AMD +boot. The stated reason for NitroTPM is that it is not confidential compute at +all — the AWS hypervisor is inside the TCB — so both need an explicit operator +opt-in on top of whatever the external auth policy decides. + +### 9.3 Do not edit a running KMS's compose to flip that flag + +The local key provider seals to +`SHA-256(SGX sealing key || MRTD || RTMR0 || RTMR1 || RTMR2 || RTMR3)`, and +**RTMR3 carries the compose hash**. Changing one line of a bootstrapped KMS's +config gives it a different sealing key, and its root CA is gone with the disk. + +Use onboarding instead: deploy a second KMS with `auto_bootstrap_domain = ""`, +then + +``` +curl -X POST http:///prpc/Onboard \ + -H 'Content-Type: application/json' \ + -d '{"source_url":"https://","domain":""}' +curl -X POST http:///prpc/Finish -d '{}' -H 'Content-Type: application/json' +``` + +The new KMS inherits the root key, so apps it serves are still trusted by a +gateway registered against the old one. **Compare the CA public key, not the CA +certificate** — the onboarded KMS re-issues its own cert, so the PEM differs +while `openssl x509 -pubkey` is identical. A KMS that bootstrapped independently +differs in both. + +### 9.4 KDS is on the critical path + +Verification needs the ASK and VCEK certificates. They come from +`https://kdsintf.amd.com`, a **single global endpoint with no mirror**, rate +limited to roughly one identical request per 10s, and unreachable for an entire +test round — TCP timeouts and 100% ICMP loss from five vantage points across four +autonomous systems and two continents, while every other external service +answered normally and DNS stayed healthy throughout. + +The rate limit is the one that will bite you mid-test: back-to-back verifications +of the same chip return `HTTP status client error (429)` on the VCEK URL, which +reads exactly like a verification failure until you notice the status code. Space +them ~25s apart. + +Caching is in-process only. `AmdKdsClient` keeps a `moka` cache of 16 CA chains +and 1024 VCEKs, capacity-bounded, no TTL, no persistence — warm for the life of +one KMS or verifier process and cold in every one-shot `dstack-verifier --verify`. +There is no KDS cache service in this repo — compare `nvidia-attest-proxy`, the +persistent on-disk cache the NVIDIA collateral got and AMD did not — but +`[core.attestation.urls] amd_kds` accepts any KDS-compatible base URL, and putting +a caching proxy there works. Four consecutive cold-process `dstack-verifier +--verify` runs of one attestation pass through a mirror; the same sequence run +directly against KDS fails on the second. A KMS configured the same way released +keys to an AMD guest end to end. + +Three things decide whether such a proxy is correct, and all three are easy to get +wrong: + +```js +export default { + async fetch(request, env, ctx) { + const url = new URL(request.url); + if (request.method !== "GET" || !url.pathname.startsWith("/vcek/v1/")) { + return new Response("not found", { status: 404 }); + } + const cache = caches.default; + const key = new Request(url.toString()); // query string carries the TCB SPLs + const hit = await cache.match(key); + if (hit) return hit; + + // 1. AMD answers 403 to an unexpected Host header, so do not forward it. + const upstream = await fetch("https://kdsintf.amd.com" + url.pathname + url.search, + { headers: { accept: "*/*" } }); + const body = await upstream.arrayBuffer(); + + // 2. Only success is cacheable. A cached 429 turns a rate limit into an outage. + if (!upstream.ok) { + return new Response(body, { status: upstream.status, + headers: { "cache-control": "no-store" } }); + } + + // 3. Every KDS response says `Cache-Control: no-cache`; override it. + const res = new Response(body, { headers: { + "content-type": "application/octet-stream", + "cache-control": "public, max-age=2592000", + }}); + ctx.waitUntil(cache.put(key, res.clone())); + return res; + }, +}; +``` + +Caching is safe here because the collateral is signature-verified downstream and +the ARK is compiled in, so a mirror can withhold or replay genuine certificates +but cannot forge one. Two fetches of one VCEK URL return different bytes — AMD +re-issues rather than serving a fixed object — but any valid copy is as good as +another, and a VCEK is valid for seven years. Bound the TTL anyway: dstack checks +no CRL and no certificate validity dates, and an unbounded cache would make that +permanent rather than merely current. + +Do not plan on serving the certificates from the host instead. The verifier side +supports it — `normalize_kernel_cert_table` reads ASK and VCEK from the SNP +extended report's certificate table — and so does the guest, but nothing can put +them there: `sev-snp-guest` in stock QEMU has no `certs-path`-style property, and +the host-side `SNP_SET_EXT_CONFIG` ioctl is not in upstream `psp-sev.h`. Guests +ask via `SNP_GET_EXT_REPORT` and get an empty table, which is why every +verification in this round went to KDS. + +A note for whoever touches this code: the ARK is already compiled in, and the ARK +fetched from KDS is discarded (`let (_fetched_ark, ask) = ...`). The network +request exists only to obtain the ASK, which is equally static per product family. +Bundling it would remove the `cert_chain` request entirely, leaving only the +per-chip VCEK. + +### 9.5 Reaching the API that holds the keys + +`GetKey` and `Attest` are **not** on the guest agent's external port. That +listener serves `Info`, `Version` and `Health` only, by design — anyone who can +route to the CVM reaches it. Everything else lives on +`/var/run/dstack.sock` inside the guest, mounted at `/`, `/v0` and `/v1` with +**no `/prpc` prefix**: the internal path is `/v1/GetKey`, not `/prpc/v1/GetKey`. + +To reach it from the host during a test, add a socat sidecar to the app compose: + +```yaml + sockproxy: + image: alpine/socat + command: TCP-LISTEN:8091,fork,reuseaddr UNIX-CONNECT:/var/run/dstack.sock + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + ports: + - "8091:8091" +``` + +That exposes app key material to the host, so it belongs in a lab and nowhere +else. + +One more encoding trap, shared with `dstack-verifier --verify`: `bytes` fields in +these JSON bodies are **hex strings**, not base64 and not byte arrays. Base64 +fails with `Invalid character 'w' at position 5`, which names the symptom and not +the cause. + +--- + +## 10. Fixed in 0.6.0-rc0 testing — do not re-diagnose + +- **KMS image had no CA certificates** and could not start at all + ([#1128](https://github.com/Dstack-TEE/dstack/pull/1128)). Failed at + `AttestationVerifier::load` with `No CA certificates were loaded from the + system`, because `reqwest` builds its clients eagerly and the AMD KDS client is + constructed even on an Intel-only host. +- **ACME issuance failed outright** ([#1129](https://github.com/Dstack-TEE/dstack/pull/1129)). + Let's Encrypt now offers `dns-persist-01` alongside `dns-01`; it carries no + `token`, and `instant-acme` 0.7.2 required that field, so the whole + authorization failed to deserialize and took the usable challenge with it. +- **DNS self-check could never pass on some zones** + ([#1130](https://github.com/Dstack-TEE/dstack/pull/1130)). The check queried the + record immediately after creating it, cached the resulting NXDOMAIN at the + recursive resolver for the zone's SOA minimum (1800s is common), and then + retried inside a 300s budget against that cache. +- **`prek` CI failed on any newly created `release/**` branch** + ([#1127](https://github.com/Dstack-TEE/dstack/pull/1127)). A branch-creating + push reports an all-zero `before` SHA, making the diff range invalid. + +If you hit any of these on a build that predates the fixes, that is why. + +--- + +## 11. Housekeeping + +- **Keep an inventory as you go**: CVMs, ufw rules, DNS records, host module and + permission changes, registry tags. Comment ufw rules so they can be found later + (`ufw allow 9350/tcp comment "dstack rc test"`). +- Host changes from section 1 (`/dev/sev` mode, `vhost_vsock`, the VMware vsock + blacklist) **revert on reboot**. Decide whether to persist them; either is fine, + but know which you chose. +- Remove test ZtDomains and stray `_acme-challenge` TXT records when finished. +- A cluster run leaves a WireGuard interface per node and a second set of ufw + rules and ports; both outlive the CVMs. +- An AMD run adds a KMS per release-gate variation. Each is a full CVM with its + own disk and port; the ones that only served as a stepping stone are safe to + delete once the onboarded KMS works, but nothing cleans them up for you. +- **Do not poll with `pgrep -f` on a pattern your own commands contain.** A + waiter looping on `pgrep -f "build-image.sh ..."` never exits while you check + progress with a command whose own command line includes that string — checking + keeps it waiting. The same self-match reports a finished build as still + running, and a bare `pgrep -af qemu-system` matches the grep itself. Check for + the artifact (`docker image inspect`, a file, a port) rather than the process. +- When doing an A/B across dependency versions, **restore `Cargo.lock` along with + `Cargo.toml`.** Editing the manifest and running cargo rewrites the lockfile; + restoring only the manifest leaves them inconsistent. CI will not catch it + unless a step passes `--locked`. +- Run any A/B with both arms on the current build. Reusing an earlier failure log + as the control is tempting and usually invalid — the image, the credentials and + the certificate state have all moved since. diff --git a/docs/security/cvm-boundaries.md b/docs/security/cvm-boundaries.md index e35496727..b0a106eef 100644 --- a/docs/security/cvm-boundaries.md +++ b/docs/security/cvm-boundaries.md @@ -35,17 +35,20 @@ This is the main configuration file for the application in JSON format: | local_key_provider_enabled | 0.3.1 | boolean | Use a local key provider | | key_provider_id | 0.5.1 | string | Optional pin for the key provider identity (hex-encoded bytes). For `kms` this is the KMS CA public key; for `local` the sealing-provider MR. For `tpm` and `none` it must be an empty string — the TPM app-root public key is instance-specific and is not used as a provider id or measured as one. | | public_logs | 0.3.3 | boolean | Whether logs are publicly visible | -| public_sysinfo | 0.3.3 | boolean | Whether system info is public | +| public_sysinfo | 0.3.3 | boolean | Whether system info is public. Covers the guest dashboard and `/metrics`, including the `dstack_gpu_*` series (since 0.6.0), which expose each GPU's UUID and PCI bus address alongside utilization, memory, temperature and power. | | public_tcbinfo | 0.5.1 | boolean | Whether TCB info is public | | allowed_envs | 0.4.2 | array of string | List of allowed environment variable names | | no_instance_id | 0.4.2 | boolean | Disable instance ID generation | | secure_time | 0.5.0 | boolean | Whether secure time is enabled | -| pre_launch_script | 0.4.0 | string | Prelaunch bash script that runs before execute `docker compose up` | -| init_script | 0.5.5 | string | Bash script that executed prior to dockerd startup | -| storage_fs | 0.5.5 | string | Filesystem type for the data disk of the CVM. Supported values: "zfs", "ext4". default to "zfs". **ZFS:** Ensures filesystem integrity with built-in data protection features. **ext4:** Provides better performance for database applications with lower overhead and faster I/O operations, but no strong integrity protection. | +| pre_launch_script | 0.4.0 | string | Prelaunch bash script that runs before `docker compose up`. It runs *after* dockerd, so containers restored by a Docker restart policy can already be running when it executes. Do not build security gates on it — see [security-best-practices.md](./security-best-practices.md#security-semantics-must-not-depend-on-pre_launch_script-running-first). | +| init_script | 0.5.5 (string), 0.6.0 (string[]) | string or string[] | Up to 5 Bash scripts executed in order prior to dockerd startup, so they always complete before any container starts, on every boot; a string is treated as a one-element array. Multiple scripts require string `manifest_version: "3"` so older guests fail closed. MrConfigV3 binds the hashes only for manifest v3. | +| storage_fs | 0.5.5 | string | Filesystem type for the data disk of the CVM. Supported values: "zfs", "ext4". default to "zfs". **ZFS:** Checksums every block, so a modified block fails the read. **ext4:** Lower overhead and faster I/O for database workloads; checksums metadata but not file data, so a modified block is not detected. | +| storage_discard | 0.6.0 | boolean | Return unused data-disk blocks to the host. Defaults to `true` so encrypted sparse images track live data instead of historical writes. Set to `false` if revealing filesystem allocation and deletion patterns to the host is unacceptable. | | swap_size | 0.5.5 | string/integer | The linux swap size. default to 0. Can be in byte or human-readable format (e.g., "1G", "256M"). | -| key_provider | 0.5.6 | string | Key provider type. Supported values: "none", "kms", "local", "tpm". `"tpm"` is only supported on platforms whose TPM is part of the platform trust model (GCP vTPM, AWS EC2 NitroTPM); on other platforms guest setup fails closed, since sealing to a host-provided software TPM (e.g. swtpm under bare QEMU) offers no protection against the host. | +| key_provider | 0.5.6 | string | Key provider type. Supported values: "none", "kms", "local", "tpm". GCP vTPM and AWS EC2 NitroTPM are part of their platform trust models. The Dstack platform can use VMM-managed swtpm for seal/unseal and restart persistence, but it offers no protection against the host and is intentionally not accepted by remote verifiers. | +The five-script limit bounds runtime-event-log and MrConfigV3 growth while +allowing several independently approved infrastructure initialization stages. The hash of this file content is extended as the dstack `compose-hash` launch event. On TDX-family platforms the launch event is measured into RTMR3. On AWS NitroTPM it is measured into non-resettable SHA384 PCR14 before the `system-ready` launch boundary. Remote verifiers extract and replay this event during attestation. @@ -74,6 +77,7 @@ This file contains system configuration in JSON format: | kms_urls | array of string | List of KMS service URLs | | gateway_urls | array of string | List of gateway service URLs | | pccs_url | string | URL of the PCCS service (used when dstack components need to verify a remote TD CVM or SGX enclave) | +| nvidia_attestation_proxy_url | string | Optional persistent OCSP and RIM cache used by NVIDIA local GPU attestation | | docker_registry | string | URL of the docker registry | | host_api_url | string | VSOCK URL of host API | | vm_config | string | JSON string of VM configuration (os_image_hash, cpu_count, memory_size) | @@ -85,6 +89,7 @@ The hash of this file is not extended to any RTMR because each field has its own | kms_urls | URLs themselves aren't security-critical. The trust anchor is the KMS root public key, which is extended as the `key-provider` launch event. On TDX-family platforms this is RTMR3; on AWS NitroTPM this is PCR14. Keys obtained from KMS will either successfully decrypt/encrypt the disk or fail-and-abort. | | gateway_urls | URLs aren't security-critical. Trust is established through CA certificates from KMS. App CVM and dstack-gateway CVM verify each other's CA certificates to ensure they're under the same KMS authority. | | pccs_url | URL isn't security-critical. Trust is anchored by the root public key pinned in the attestation verification program. | +| nvidia_attestation_proxy_url | The URL is not a collateral trust anchor. The measured guest verifies NVIDIA signatures and the signed OCSP validity window, and continues to require a fresh GPU evidence nonce. A bad endpoint can withhold collateral and cause a denial of service, but cannot forge a successful attestation or replay an expired `good` response. | | docker_registry | Docker daemon verifies image integrity using the pinned image hashes in the docker-compose file. | | host_api_url | Used only for reporting or encrypted sealing key transport. An incorrect URL doesn't create security vulnerabilities. | | vm_config | Informs the CVM to report virtual hardware info to KMS when requesting keys. KMS uses this info to calculate expected RTMRs and verify image hash. If tampered with, image hash verification would fail and no keys would be distributed. | @@ -176,17 +181,72 @@ Full specification: [host_api.proto](../../dstack/host-api/proto/host_api.proto) The dstack-guest-agent runs an HTTP server on port 8090 inside the CVM. This port is publicly accessible, allowing external clients to view basic CVM information. -| Service | Purpose | -|---------|--------| -| Worker | Provides public-facing app information | +Since dstack 0.6.0 the listener serves two API surfaces, selected by URL path +alone: the frozen v0.5.11 `Worker` service at `/prpc` (equivalently +`/prpc/v0`), and the versioned `dstack.guest.v1` `Worker` service at +`/prpc/v1`. The frozen surface is closed and never changes again; new +capability arrives only on v1. [guest-api-v1.md](../guest-api-v1.md) is the +normative specification of the v1 surface, including its status-code and +version-probing rules. -**Available Methods:** +Neither surface returns key material, and no caller chooses what gets signed +or attested. That boundary, not the method list, is what makes this listener +safe to expose: key material and caller-chosen attestation live only on the +internal Unix socket (`/var/run/dstack.sock`), which is not a CVM boundary — +it is reachable only by the application itself. An application that re-exports +that socket has moved the boundary itself, and everything behind it moves with +it. + +**Frozen `Worker` (`/prpc`, alias `/prpc/v0`):** | Method | Description | Return Type | |--------|-------------|------------| | Info | Get application information | AppInfo | | Version | Get guest agent version | WorkerVersion | +| GetAttestationForAppKey | Attest the key the agent derives for the app | GetQuoteResponse | + +**v1 `Worker` (`/prpc/v1`):** + +| Method | Description | Return Type | +|--------|-------------|------------| +| Info | Get application identity, plus configuration when `public_tcbinfo` is set | InfoResponse | +| Version | Get guest agent version | VersionResponse | +| Health | Report whether the application is serving | HealthResponse | + +Everything on this listener is unauthenticated, so each method is bounded in +what it costs and in what it says: + +- `Health` (v1) answers from a cache the agent refreshes on its own timer, so a + call costs a lock and a clone however many callers there are. It reveals + whether the app opted into health gating, its current verdict, and — when the + app declared a `health_status_file` — the path it named and which parsing + rule failed. That last part is a narrow oracle for whether a path exists and + what shape its first two lines have; the path itself is already public, since + it is measured into the compose hash. The file's *contents* are never quoted + back. Container names and statuses were already public through the dashboard + below. +- `GetAttestationForAppKey` (frozen) generates a fresh platform attestation per + call. With the frozen `Info` below, it is one of the two methods here that + let an anonymous caller drive quote generation. It has no v1 counterpart + on purpose: a v1 application attests its own key through the internal socket + (`/v1/GetKey`, then `/v1/Attest`) and serves the result itself, so the public + listener never gained a second attestation-on-demand entry point. +- The frozen `Info` decodes identity out of a boot attestation per call, which + costs a hardware quote under the agent's global quote lock. The v1 `Info` + serves the same identity from a cache decoded once at startup, so an + anonymous caller cannot drive quote generation through it; if the boot-time + decode failed, retries are throttled to one attempt per interval. Of the two + quote-generating methods, the frozen `Info` is the one worth rate-limiting + first: it is what clients actually poll, and it replays the event log on top + of the quote. +- Both `Info` methods honour the app's `public_tcbinfo` choice, with different + reach. The frozen one blanks `tcb_info` and `vm_config` but always serves + `key_provider_info`. The v1 one blanks `app_compose`, `vm_config`, and + `key_provider_info`, and carries no measurement registers or event log at + all — those are attestation data and belong to the internal `Attest`, where + a quote vouches for them. Identity and the measurement hashes are always + visible on both surfaces. The service also provides a web dashboard at the root URL (`/`) showing basic CVM information. View the dashboard template [here](../../dstack/guest-agent/templates/dashboard.html). -Full specification: [agent_rpc.proto](../../dstack/guest-agent/rpc/proto/agent_rpc.proto) +Full specifications: [agent_rpc.proto](../../dstack/guest-agent/rpc/proto/agent_rpc.proto) for the frozen surface, [agent_rpc_v1.proto](../../dstack/guest-agent/rpc/proto/agent_rpc_v1.proto) for v1. diff --git a/docs/security/public-security-reports.md b/docs/security/public-security-reports.md index 06a8ce690..d04b40628 100644 --- a/docs/security/public-security-reports.md +++ b/docs/security/public-security-reports.md @@ -37,7 +37,7 @@ These issues were filed as concrete vulnerability reports, security audit findin | [#558](https://github.com/Dstack-TEE/dstack/issues/558) Path traversal in KMS `remove_cache` | Closed | Valid report, fixed | [#601](https://github.com/Dstack-TEE/dstack/pull/601) validates cache paths before deletion | | [#559](https://github.com/Dstack-TEE/dstack/issues/559) Zero `mr_config_id` bypasses verification and weakens `mr_aggregated` identity | Closed | Not a production vulnerability | Zero `mr_config_id` remains an unset-value compatibility case, and configuration changes are still reflected through RTMR-based measurements. No code fix was applied | | [#560](https://github.com/Dstack-TEE/dstack/issues/560) Admin token comparison not constant-time | Closed | Not a production vulnerability | The comparison is over a SHA-256 digest of a high-entropy token, not the raw token. No code fix was applied | -| [#561](https://github.com/Dstack-TEE/dstack/issues/561) KMS TLS client certificates are non-mandatory in Rocket config | Closed | Valid report, documented | The TLS listener allows unauthenticated bootstrap, temp-CA bootstrap, and public endpoints. `GetTempCaCert` returns temp CA private material for bootstrap. App/KMS key release requires verified caller attestation, and certificate signing verifies the CSR signature and embedded attestation. No code fix was applied | +| [#561](https://github.com/Dstack-TEE/dstack/issues/561) KMS TLS client certificates are non-mandatory in Rocket config | Closed | Valid report, fixed | [#1106](https://github.com/Dstack-TEE/dstack/pull/1106) makes the KMS verify client certificates by the attestation they carry instead of by an issuer CA, so a presented certificate must now be attested at the handshake. The listener still allows unauthenticated bootstrap and public endpoints. App/KMS key release requires verified caller attestation, and certificate signing verifies the CSR signature and embedded attestation | | [#562](https://github.com/Dstack-TEE/dstack/issues/562) Configfs path overridable through an environment variable | Closed | Not a production vulnerability | A process that can choose its own quote path is already inside the measured CVM behavior. No code fix has been applied. A production guard for `DCAP_TDX_QUOTE_CONFIGFS_PATH` remains possible hardening | | [#563](https://github.com/Dstack-TEE/dstack/issues/563) `simulate_quote` runtime path in production guest agent | Closed | Valid report, fixed | [#582](https://github.com/Dstack-TEE/dstack/pull/582) isolates the simulator into a dedicated binary | | [#564](https://github.com/Dstack-TEE/dstack/issues/564) `GetAppEnvEncryptPubKey` unauthenticated app ID enumeration | Closed | Not a production vulnerability | The RPC returns a public encryption key before an app has an attested identity, and `app_id` is not treated as secret. No code fix was applied | @@ -49,7 +49,7 @@ These issues were filed as concrete vulnerability reports, security audit findin | [#606](https://github.com/Dstack-TEE/dstack/issues/606) App keys and decrypted env files world-readable | Open | Valid hardening, open | Tightening secret-bearing file writes to owner-only permissions (`0600`) is valid defense-in-depth work with no expected compatibility cost | | [#607](https://github.com/Dstack-TEE/dstack/issues/607) `gateway_app_id = "any"` disables gateway identity pinning | Closed | Not a production vulnerability | `gateway_app_id` is KMS contract configuration and is publicly auditable. Production deployments must not use `"any"`. No code fix was applied | | [#608](https://github.com/Dstack-TEE/dstack/issues/608) `auth_api.type = "dev"` allows all authorization | Closed | Not a production vulnerability | Dev auth is measured runtime configuration, not a production mode. Production must use webhook/on-chain authorization. No code fix was applied | -| [#609](https://github.com/Dstack-TEE/dstack/issues/609) `quote_enabled = false` bypasses attestation | Closed | Not a production vulnerability | The flag is measured in runtime configuration and should fail production attestation policy. No code fix was applied | +| [#609](https://github.com/Dstack-TEE/dstack/issues/609) `quote_enabled = false` bypasses attestation | Closed | Not a production vulnerability | The flag was measured in runtime configuration and would fail production attestation policy, so no code fix was applied at the time. The setting has since been retired; current KMS uses `attest_rpc_cert`, which defaults to `true`, to control RPC certificate attestation | | [#610](https://github.com/Dstack-TEE/dstack/issues/610) Unauthenticated bootstrap endpoint can overwrite root keys | Closed | Not a production vulnerability | The bootstrap endpoint does not accept caller-supplied root key material. Root keys are generated server-side, and the operator chooses which result to publish. No code fix was applied | | [#611](https://github.com/Dstack-TEE/dstack/issues/611) Unauthenticated `/finish` endpoint can shut down KMS onboard service | Closed | Not a production vulnerability | The onboard service is a short-lived setup flow. Premature shutdown causes operator retry, not persistent compromise or data loss. No code fix was applied | | [#612](https://github.com/Dstack-TEE/dstack/issues/612) Gateway `register_cvm` prefers stale `app_info` over live attestation | Closed | Not a production vulnerability | Cert-embedded `app_info` is extracted from attestation and signed by KMS. Preferring it avoids redundant extraction and is not a trust bypass. No code fix was applied | @@ -59,7 +59,7 @@ These issues were filed as concrete vulnerability reports, security audit findin | [#616](https://github.com/Dstack-TEE/dstack/issues/616) Host-controlled Docker registry mirror enables image substitution attacks | Closed | Not a production vulnerability | Registry mirrors are untrusted transport. Digest-pinned image references and measured compose configuration protect against substitution. No code fix was applied | | [#617](https://github.com/Dstack-TEE/dstack/issues/617) Guest agent exposes raw private keys to all local processes | Closed | Not a production vulnerability | dstack treats a CVM as one application trust domain. It does not provide per-container key isolation inside the same measured application. No code fix was applied | | [#618](https://github.com/Dstack-TEE/dstack/issues/618) Disk encryption disableable via kernel cmdline, not measured in RTMR | Closed | Not a production vulnerability | The kernel command line is measured into RTMR2, so changing `dstack.storage_encrypted=false` changes attestation evidence. No code fix was applied | -| [#619](https://github.com/Dstack-TEE/dstack/issues/619) KMS `get_temp_ca_cert` returns temp CA private key without authentication | Closed | Duplicate | The report duplicates the private advisory response for the temp CA bootstrap flow | +| [#619](https://github.com/Dstack-TEE/dstack/issues/619) KMS `get_temp_ca_cert` returns temp CA private key without authentication | Closed | Duplicate | The report duplicates the private advisory response for the temp CA bootstrap flow. [#1106](https://github.com/Dstack-TEE/dstack/pull/1106) removes the reason the temp CA existed - the KMS no longer pins it - so callers can be migrated off it; the RPC itself is retained until guests and KMS-to-KMS onboarding are | ## Related security roadmap and hardening diff --git a/docs/security/security-best-practices.md b/docs/security/security-best-practices.md index af3f94263..91b3c0c2e 100644 --- a/docs/security/security-best-practices.md +++ b/docs/security/security-best-practices.md @@ -60,7 +60,29 @@ You only need to add the `APP_LAUNCH_TOKEN` environment variable to enable LAUNC ![Token Environment Variable](../assets/token-env.png) -user_config is not encrypted, and similarly requires integrity checks at the application layer. For example, you can store a USER_CONFIG_HASH in encrypted environment variables and verify it in the prelaunch script. +`user_config` is not encrypted, and similarly requires integrity checks at the application layer. For example, you can store a `USER_CONFIG_HASH` in encrypted environment variables and verify it in the `pre_launch_script`. Such a check is a defense-in-depth measure, not a gate: it does not reliably run before your containers do (see the next section), so the authoritative check belongs in `init_script` or in the application itself. + +## Security semantics must not depend on `pre_launch_script` running first + +`pre_launch_script` runs from `app-compose.service`, which is ordered `After=docker.service`. Docker restores containers when the daemon starts, so on a reboot your application can already be running by the time the prelaunch script executes: + +- **`restart: always`** — Docker restarts the container whenever the daemon starts, even if it was stopped cleanly beforehand. Every reboot takes this path. This is the restart policy used in several dstack examples. +- **`restart: unless-stopped`** — a clean shutdown runs the `ExecStop` of `app-compose.service`, which stops the containers, so Docker does not restore them. But an unclean stop (host reset, guest crash, power loss) leaves them in the running state and Docker restores them on the next boot. The host decides when to reset a CVM, so it can force this path at will. + +`init_script` has no such gap. It runs from `dstack-prepare.service`, which is ordered `Before=docker.service`, so every init script completes before dockerd — and therefore before any container — starts, on every boot. Init scripts also run after `dstack-util setup`, so app keys and the decrypted env file are already available to them. Anything that must run before application code belongs in `init_script`. + +This is an ordering property, not an integrity one. Both scripts are measured into the compose hash, so the scripts you audited are the scripts that run. What is not guaranteed is that the prelaunch one runs *first*. + +**Unsafe in `pre_launch_script`:** + +- Verifying `USER_CONFIG_HASH` or an `APP_LAUNCH_TOKEN` and calling `exit 1` to abort the launch. After a reboot the app is already serving with the unverified input; the non-zero exit only marks `app-compose.service` as failed. +- Fetching or integrity-checking a data file, model weights, or a database snapshot before the app consumes it. The restored container may already have read the previous, unchecked copy. +- Installing firewall rules, network namespaces, or an egress proxy that is meant to contain the application. There is a window in which the app runs unconstrained. +- Deriving or writing a secret that the app expects to find on disk. On the early-start path the app sees whatever the previous boot left there. + +**Safe in `pre_launch_script`:** work whose only effect is on the `docker compose up` that immediately follows it — pre-pulling or importing images, generating a compose override, or writing files that containers pick up only when that compose run recreates them. + +**For app auditors:** treat any check in a `pre_launch_script` as advisory. When judging whether a deployment enforces a security property, ask whether the property still holds on a boot where the prelaunch script has not run yet. If it does not, the check must move into `init_script`, or into the application itself before it serves traffic or touches secrets. ## Don't put secrets in docker-compose.yaml @@ -84,15 +106,25 @@ Example app-compose.json: Development settings are intentionally easy to audit, but they are not production-safe. A production deployment should satisfy all of the following: -- KMS quote verification remains enabled. Do not deploy production KMS with `quote_enabled = false`. +- The KMS attests its own RPC certificate. Do not deploy production KMS with `attest_rpc_cert = false`. - KMS authorization uses webhook/on-chain policy. Do not use `auth_api.type = "dev"` with real key material. - The KMS contract pins a concrete gateway app id. Do not use `gateway_app_id = "any"` for production traffic. - TEE quotes are evaluated by deployment policy, including TCB status and expected OS/application measurements. -The KMS TLS listener may keep `rpc.tls.mutual.mandatory = false` because bootstrap, temp-CA bootstrap, and public metadata endpoints need to be reachable before a client has an RA-TLS certificate. `GetTempCaCert` returns temp CA private material for the bootstrap flow; treat it as bootstrap-sensitive. +The KMS TLS listener verifies client certificates by the attestation they carry rather than by an issuer CA, so it needs no `rpc.tls.mutual` section. It still accepts connections without a client certificate, because bootstrap and public metadata endpoints must be reachable before a client has an RA-TLS certificate. `GetTempCaCert` remains in use by guests and by KMS-to-KMS onboarding, which still mint their client certificates from that CA; it returns temp CA private material, so treat it as bootstrap-sensitive. App key release and KMS key handover still require verified caller attestation from the RA-TLS client certificate. Certificate signing verifies the CSR signature and embedded attestation before signing. +## Management/admin API authentication + +The VMM, gateway, and KMS management surfaces must have authentication enabled in production: + +- VMM: set `[auth] enabled = true` with `tokens` (or `htpasswd_file`) — this guards the entire VMM HTTP/pRPC/UI surface. Never bind to a non-localhost address without it. Clients send `Authorization: Bearer ` or `X-Admin-Token`. +- Gateway: set `[core.admin] admin_token` (or `htpasswd_file`) and keep `insecure_no_auth = false`. Clients send `Authorization: Bearer ` or `X-Admin-Token`. +- KMS: enable `[core.admin]` with an `auth_token` (or `htpasswd_file`); the admin RPCs are served on a dedicated listener and clients send `Authorization: Bearer ` or `X-Admin-Token`. Enabled with no credential denies all admin RPCs (fail-closed). + +All three share the same HTTP authenticator: bcrypt-only htpasswd (via `htpasswd -B`), constant-time token comparison, and fail-closed behavior. + ## Keep private material owner-only Secret-bearing files should be owner-only (`0600`) wherever possible, including app keys, decrypted env files, KMS root keys, gateway WireGuard/TLS keys, and ACME credentials. Preserve restrictive permissions when copying volumes, backing up `/etc/kms/certs`, or moving gateway and certbot state between hosts. Public issue [#606](https://github.com/Dstack-TEE/dstack/issues/606) tracks the remaining low-cost hardening work in dstack-managed file writes. @@ -152,3 +184,23 @@ services: image: nginx@sha256:eee5eae48e79b2e75178328c7c585b89d676eaae616f03f9a1813aaed820745a network_mode: host ``` +## Runtime event-log V2 policies + +Event-log V2 exposes canonical digest pre-images so a relying party can check +individual claims such as `compose-hash`. Verifying +`sha384(preimage) == digest` proves only that those bytes participate in the +quoted RTMR/PCR extension chain. It does **not** prove that trusted dstack boot +code originated the event name: privileged code inside the CVM can append +additional measured events after boot. + +Policies that trust a named V2 event must therefore also validate ordering and +the boot boundary. In particular, select the expected claim before +`boot-mr-done`/`system-ready`, reject duplicate trusted claim names, and replay +the complete quoted chain. Never accept an arbitrary later event solely because +its digest matches its supplied pre-image. + +V2 is a coordinated upgrade. Upgrade every KMS, gateway, verifier, and other +relying party before enabling `event_log_version: 2`; older verifiers interpret +runtime events as V1 and reject the quote. Older guest images may ignore the +compose field and emit V1 events, so confirm that the selected image advertises +V2 support before relying on per-event claims. diff --git a/docs/security/security-model.md b/docs/security/security-model.md index b85a83bf3..751dbdc79 100644 --- a/docs/security/security-model.md +++ b/docs/security/security-model.md @@ -8,7 +8,7 @@ This document helps you evaluate whether dstack's security model fits your needs dstack removes the need to trust most infrastructure operators. On TEE platforms such as Intel TDX and AMD SEV-SNP, the cloud or host operator cannot read your protected memory, modify your measured code, or access your secrets without detection. On AWS EC2 NitroTPM attested instances, AWS Nitro is part of the trusted platform, and the untrusted party is the workload AWS account administrator/operator. Network attackers cannot intercept your traffic because TLS terminates inside the attested environment with keys controlled by that environment (Zero Trust HTTPS). Docker registries cannot serve malicious images because the guest verifies SHA256 digests before pulling. -The primary trust input is **attested platform hardware**. Intel TDX is the production TEE path. AMD SEV-SNP is available where the selected dstack OS image and host support it, but it is new and experimental. AWS EC2 NitroTPM attested instances use a different trust root: the AWS Nitro system and AWS NitroTPM attestation PKI. In that mode, the threat model protects against the workload AWS account administrator and EC2 operator actions, but AWS remains trusted. For GPU workloads, you also trust **NVIDIA GPU hardware** and NVIDIA's Remote Attestation Service (NRAS). These are hardware-level trust assumptions. +The primary trust input is **attested platform hardware**. Intel TDX is the production TEE path. AMD SEV-SNP is available where the selected dstack OS image and host support it, but it is new and experimental. AWS EC2 NitroTPM attested instances use a different trust root: the AWS Nitro system and AWS NitroTPM attestation PKI. In that mode, the threat model protects against the workload AWS account administrator and EC2 operator actions, but AWS remains trusted. For GPU workloads, you also trust **NVIDIA GPU hardware**, NVIDIA's attestation PKI/RIMs, and its revocation service (or NRAS when a deployment selects remote verification). These are hardware-level trust assumptions. Everything else is verifiable. @@ -24,6 +24,8 @@ dstack protects the execution environment, not your application code. Bugs in yo Infrastructure operators can still deny service. They can shut down your workload, throttle resources, or block network access. If availability matters, plan for redundancy across providers. +**Persistent-storage freshness, integrity, and availability.** Disk encryption protects the confidentiality of data at rest, not its integrity: the LUKS2 volume carries no authentication tag, so detecting a modified block is left to the filesystem. The default ZFS checksums every block and fails the read; with `storage_fs` set to ext4, file data is not checksummed and a modified block reaches the application. Neither filesystem proves that an attached disk represents the latest application state. An infrastructure operator can withhold, delete, replace, or restore an earlier valid encrypted disk image. Applications that require rollback-resistant state must anchor a monotonic version or state commitment in an external trusted service, ledger, or equivalent freshness mechanism. + ## Security Guarantees ### Confidentiality @@ -55,11 +57,110 @@ dstack supports NVIDIA H100, H200, and B200 GPUs in confidential compute mode fo ### How It Works -GPUs are passed through via VFIO directly to the TEE-protected CVM. The GPU operates in confidential compute mode, encrypting data during computation. Both the CPU TEE and NVIDIA GPU provide hardware isolation together. If either component fails verification, the security model breaks. +GPUs are passed through via VFIO to the TEE-protected CVM. Before key provisioning, `dstack-util setup` inventories every VGA/3D-controller PCI function, rejects non-NVIDIA display devices, and runs NVIDIA's local `nvattest` verifier over every NVIDIA-driver-visible GPU with a fresh nonce. dstack does not ship or pass a custom relying-party policy to this command; nvattest's built-in appraisal still applies. The complete JSON result (`result_code`, `result_message`, `claims`, and `detached_eat`) is saved at `/run/nvidia-gpu-attestation/attestation.out`. + +An application may additionally set `requirements.gpu_policy` to an object with the following deny-unknown-fields schema: + +- `attest_gpu` (boolean, default `true`): require local NVIDIA GPU attestation before enabling an attached GPU. Setting this to `false` skips attestation and is an explicit reduction in protection. +- `rego` (optional string): a Rego v0 script evaluated with the nvattest output's `claims` array as `input`. It must define the boolean entrypoint `data.policy.nv_match` in `package policy`. +- `allow_devtools` (boolean, default `false`): permit NVIDIA DevTools mode. Production applications should leave this disabled because DevTools removes the expected GPU memory-confidentiality guarantee. +- `allow_debug` (boolean, default `false`): permit an attestation claim whose `dbgstat` is `enabled`. +- `allow_insecure_boot` (boolean, default `false`): permit an attestation claim whose GPU `secboot` value is false. + +The optional Rego v0 policy can enforce deployment-specific claims. A minimal `app-compose.json` containing one looks like this; replace the placeholder with the policy source: + +```json +{ + "manifest_version": "3", + "name": "gpu-app", + "runner": "docker-compose", + "requirements": { + "gpu_policy": { + "rego": "" + } + } +} +``` + +For example, the following policy requires exactly one H100 whose `hwmodel` matches the value emitted by `nvattest`: + +```rego +package policy +default nv_match = false + +nv_match { + count(input) == 1 + input[0].hwmodel == "GH100 A01 GSP BROM" +} +``` + +The policy must define the boolean rule `data.policy.nv_match`. Its `input` is the complete `claims` array from `nvattest`; when no attestation is performed, `input` is `[]`, so the example also rejects a launch without exactly one attested GPU. + +After measuring `compose-hash`, dstack enters the GPU setup gate and JCS-canonicalizes the original `requirements.gpu_policy` JSON value, then measures its SHA-256 digest in a `gpu-policy-hash` event. When the field is absent—including when `requirements` itself is absent—both parsing and measurement use the default empty object `{}`. Thus an omitted policy and an explicit `{}` have the same digest, while any explicitly present field, including an explicit default value, changes the digest. MrConfigV3 GPU launches also carry this digest as the optional `gpu_policy_hash` field; non-GPU launches omit it for compatibility. When the field is present, the guest compares it with the digest computed from app-compose; when it is absent, the guest skips this MrConfigV3 check. The MrConfigV3 document is bound by TDX `MR_CONFIG_ID` or SEV-SNP `HOST_DATA`, so the host cannot substitute a different GPU policy when this optional binding is present without changing the platform launch identity. The typed policy used for enforcement applies omitted-field defaults and rejects unknown fields. If GPU attestation is enabled and NVIDIA GPUs are present, dstack attests them and applies the basic settings and optional Rego policy before setting the GPU ready state. When no attestation claims are produced—because no GPU is attached or `gpu_policy.attest_gpu` is false—Rego is still evaluated with an empty array as `input` before any ready-state transition. This lets an application reject a launch whose attested GPU count is wrong. A false, undefined, malformed, or non-boolean Rego result stops boot before key provisioning. + +Up to five init scripts may be configured. Their ordered SHA-256 digests are +measured as `init-script-hash` runtime events on platforms with a quoted +runtime register. MrConfigV3 also carries the ordered digest list; on SEV-SNP, +the signed report's `HOST_DATA` binds the exact canonical MrConfigV3 document. +An omitted `init_script_hashes` field disables this check, +while an explicit empty list requires app-compose to contain no init scripts. +When present, the guest compares the list with hashes computed from app-compose +before continuing boot. Current VMMs serialize the field explicitly for +manifest v3 launches, including `init_script_hashes: []` when the list is empty. + +The policy digest is remotely verifiable on each supported platform, but through different carriers: + +- **TDX:** `gpu-policy-hash` contains the raw 32-byte digest and is measured into RTMR3. Replay the event log and compare the result with the quote's RTMR3, then compare the event payload with the expected digest. When an MrConfigV3 document includes `gpu_policy_hash`, TDX `MR_CONFIG_ID` additionally binds that field. +- **AWS NitroTPM:** the same `gpu-policy-hash` event is extended into non-resettable SHA384 PCR14. Replay the PCR14 event chain against the signed NitroTPM Attestation Document, then compare the payload with the expected digest. +- **AMD SEV-SNP:** there is no quote-bound runtime event register in the current stack. Instead, GPU launches put the digest in optional `MrConfigV3.gpu_policy_hash`, and the signed SNP report's `HOST_DATA` binds the exact MrConfigV3 document. Verify the SNP report and `HOST_DATA` binding, then compare the field with the expected digest. If the optional field is absent, this check is not asserted. + +For every NVML-enumerated GPU, dstack calls `Device::is_cc_enabled()` and `Device::is_cc_dev_mode_enabled()` and requires the NVML device count to match the expected GPU count. CC must always be ON; DevTools must be OFF unless the measured policy explicitly permits it. The typed claim checks always require `measres == "success"`; by default they also require `dbgstat == "disabled"` and `secboot == true`, with the latter two checks controlled by their explicit opt-ins. Only after the default appraisal, typed claim checks, optional Rego policy, and per-device NVML checks succeed does dstack call `Device::set_confidential_compute_state(true)` to set the GPU ready state. The dstack CPU/guest boot chain is verified independently through measured boot; a GPU claim named `secboot` refers to the GPU appraisal, not UEFI Secure Boot in the CVM. ### Dual Attestation -GPU workloads require verification of both hardware components. The CPU TEE provides the quote that verifies CPU and memory isolation. NVIDIA's Remote Attestation Service (NRAS) independently verifies the GPU is genuine and running in confidential mode. Both attestations must pass for complete verification. +GPU workloads require verification of both hardware components. The CPU TEE quote verifies the CVM and its measured guest code. NVIDIA-signed evidence, checked against NVIDIA RIMs and certificate status by `nvattest`, verifies the GPU appraisal. After the optional policy and ready-state operations succeed, dstack emits a `gpu-attestation` launch event before `system-ready`. Its versioned payload records the number of appraised devices, asserted CC/DevTools state, and SHA-256 of the complete nvattest JSON output (claims and detached EAT). On TDX, both `gpu-policy-hash` and `gpu-attestation` are append-only RTMR3 events; they are never derived from application-controlled `report_data`. + +For a successful TDX GPU launch, the GPU-relevant RTMR3 event order is: + +```text +compose-hash +init-script-hash (zero or more, in configured order) +gpu-policy-hash +gpu-attestation +instance-id +boot-mr-done +``` + +`gpu-policy-hash` is emitted even for a GPU-less launch. `gpu-attestation` is emitted only after an attached GPU passes `nvattest`, the built-in checks, the optional Rego policy, and the NVML state checks. Its UTF-8 JSON payload has this shape: + +```json +{ + "version": 2, + "provider": "nvidia", + "devices": 1, + "cc_mode": "on", + "devtools": false, + "evidence_sha256": "" +} +``` + +`/v1/Attest` returns that complete boot-time `nvattest` record when the request sets `include_boottime_gpu_evidence`; it runs no new attestation. `AttestResponse.boottime_gpu_evidence` is a list of `GpuEvidenceBundle`, each carrying `vendor`, `format`, and `evidence`. The boot record is the bundle whose `vendor` is `nvidia` and whose `format` is `nvidia-nvattest-boottime-json-v1`; its `evidence` is hex-encoded bytes that decode to the exact UTF-8 nvattest output as the agent read it from disk. The other format, `nvidia-nvattest-collect-evidence-json-v1`, comes from `/v1/AttestGpu` and is fresh evidence against a caller nonce; the two are deliberately distinct because a verifier for one does not appraise the other. To bind the API result to TDX evidence: verify the quote, replay the event log to the quote's RTMR3, require exactly one pre-`system-ready` `gpu-attestation` event, decode its JSON payload, select the boot-time bundle by `format`, and compare `evidence_sha256` with `SHA-256(hex_decode(bundle.evidence))`. Hash the decoded bytes, not the JSON string as returned and not a re-serialized form: parsing and re-serializing changes the digest and breaks the comparison. Only after this comparison should the verifier inspect the returned claims. This exact-byte comparison includes any whitespace or trailing newline in the decoded record. + +A verifier must replay the measured event log, require exactly one `gpu-policy-hash` event immediately after `compose-hash`, and compare its 32-byte payload with the expected policy digest (`SHA-256(JCS({}))` for the omitted/default policy). When MrConfigV3 includes `gpu_policy_hash`, it must match the same digest. When GPU protection is required, the verifier must also require exactly one pre-`system-ready` `gpu-attestation` event with `devices > 0` and, when applicable, the expected deployment count. The raw `attestation.out` file is not trusted by itself; if it is supplied for inspection, its digest must match the `gpu-attestation` event. + +### GPU Threat Model and Lifetime + +The GPU gate assumes a malicious host/VMM and untrusted host-provided PCI topology, while trusting the CPU TEE, the measured dstack guest/kernel, NVIDIA hardware/firmware roots, and the cryptography used by both attestation chains. Availability is out of scope. + +The events make the following **boot-time** statement: immediately before key provisioning, all attached VGA/3D PCI functions were NVIDIA devices, their count matched the NVML inventory, nvattest returned one fresh successfully appraised claim for each device, the measured application policy accepted those claims and GPU state when present, every enumerated GPU passed the CC/DevTools NVML checks, and setting the GPU ready state succeeded. This closes these cases: + +- A GPU-less launch cannot be presented as a GPU-verified launch because it has no `gpu-attestation` event. +- A mixed launch cannot attest only its TEE-capable subset. Non-NVIDIA display GPUs are rejected, and the sysfs, NVML, and nvattest claim counts must all agree. A non-CC NVIDIA GPU either prevents evidence collection/appraisal or causes the default appraisal, application policy, or CC-state check to fail. +- Copying another CVM's result into a file or `report_data` does not work. Only measured pre-application code can place the event before `system-ready`, and event-log replay binds it to the quoted RTMR/PCR value. + +This is **not a lifetime or physical co-location guarantee**. After `system-ready`, an application with sufficient guest privileges can unload the NVIDIA driver, and a malicious host may attempt PCI hot-remove/replacement or proxy GPU traffic. The boot event remains a true historical statement but does not prove that the same device is still attached. dstack also cannot rule out a live relay/cuckoo attack to a genuine remote GPU: current Hopper/Blackwell deployments do not provide a CPU-TEE-verifiable TEE-I/O/TDISP device binding. Applications that mutate the driver or PCI topology are outside this guarantee; higher-assurance deployments must prevent that behavior and re-attest before using a newly initialized GPU; the guest-agent `/v1/AttestGpu` API does that re-check against a caller-chosen nonce, returning bundles tagged `nvidia-nvattest-collect-evidence-json-v1` rather than the boot-time format. Being an NVIDIA report, its result is subject to the same relay caveat as any attestation-time GPU sample: it establishes that a genuine CC-enabled GPU is reachable and responsive now, not that the device is bound to this TD, so it must not be forwarded to a remote relying party as proof of GPU possession. + +AMD SEV-SNP has no runtime measurement register in the current dstack stack. The local boot gate can still fail closed, but a `gpu-attestation` event carried beside an SNP report is not remotely bound to that report and must not be accepted as dual-attestation evidence. SNP needs a measured vTPM/PCR channel before it can provide the same remote binding. ### AI Workload Protection @@ -155,6 +256,15 @@ Use this checklist to verify a workload running in a dstack CVM. - [ ] Launch event log replays correctly (RTMR3 on TDX-family platforms, PCR14 on AWS NitroTPM) - [ ] Config commitment matches the expected app/config target (on AWS: PCR14 replay; PCR8 is an optional shortcut — see the [AWS verifier runbook](../aws-ec2-production-verifier-runbook.md)) - [ ] reportData contains your challenge (replay protection) +- [ ] No security-relevant check depends on `pre_launch_script` running before the application; such checks belong in `init_script` or in the application itself + +**GPU verification (when required):** +- [ ] The `gpu-attestation` device count is greater than zero and matches the expected deployment +- [ ] Exactly one `gpu-policy-hash` event follows `compose-hash`, and its payload matches `SHA-256(JCS(requirements.gpu_policy))` (default `{}` when omitted) +- [ ] When MrConfigV3 includes `gpu_policy_hash`, it matches the same expected GPU policy digest +- [ ] Exactly one `gpu-attestation` event appears before `system-ready` +- [ ] CC is ON and the event's DevTools field complies with the measured policy +- [ ] The platform binds the event log to a quoted RTMR/PCR (do not accept it from current SEV-SNP evidence) **Key management verification:** - [ ] key-provider matches expected KMS identity @@ -172,30 +282,79 @@ For TDX evidence, the event log shipped alongside an attestation is stripped dow The reason boot-time event log entries are not the verifier contract is that downstream policy compares boot measurements directly to independently reproduced expected measurements. Keeping full boot event logs would bloat evidence and expose extra detail without adding verification capability. Application identity events, by contrast, include deployment-specific values such as compose-hash, key-provider, instance-id, and runtime events. Their event log is the data a verifier needs to prove what was extended into the application measurement lane. -### Why TDX lite mode does not validate ACPI table contents - -TDX lite mode verifies the OS image without downloading the image and without -running QEMU to regenerate ACPI tables. It still uses the three RTMR0 `ACPI -DATA` digests from the attestation event log as measurement inputs. The guest -labels those three events as `acpi-loader`, `acpi-rsdp`, and `acpi-tables` -before exposing the event log, and the verifier checks that the recomputed RTMR -values match the hardware-signed quote. What it does not do is reconstruct and -byte-compare the full ACPI table contents. - -This is safe for dstack's threat model because ACPI tables are treated as -untrusted host-provided platform description, not as trusted guest code. The -dangerous executable part of ACPI is AML (ACPI Machine Language): malicious AML -can try to use `SystemMemory` operation regions through the Linux ACPICA -interpreter to read or write guest physical memory. dstack kernels include the -BadAML sandbox patch (`0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch`), -which hooks the ACPI `SystemMemory` region handler, walks the guest page tables, -and denies AML access to encrypted/private guest RAM. AML can only access -unencrypted/shared mappings. - -Therefore, an infrastructure operator can still provide bad ACPI data and cause -misconfiguration or denial of service, but unvalidated ACPI/AML cannot tamper -with confidential private memory or extract secrets. That residual availability -risk is already outside dstack's confidentiality/integrity guarantees. +### Why ACPI table verification fails closed on both TDX paths + +RTMR0 covers the three ACPI blobs QEMU hands to OVMF (`acpi-loader`, +`acpi-rsdp`, `acpi-tables`). Both TDX paths regenerate those blobs from the VM +shape declared in `vm_config` and require the recomputed digests to equal the +ones the event log reports, then rebuild the expected RTMR0 from the recomputed +values — so the expected measurement depends on nothing the host asserted about +the table contents. + +TDX lite mode did not always do this. It used to replay the three reported +digests as measurement inputs, which made RTMR0 reconstruct consistently while +leaving the table contents unconstrained. Regenerating them required running +QEMU, which the lite path exists to avoid; once ACPI generation became a pure +in-process Rust implementation, the reason for the exception disappeared. + +Verification is mandatory rather than a reported outcome because the inputs are +host-declared. `swtpm` and `qemu_version` in `vm_config` are asserted by the +untrusted host and are not independently constrained by any other measurement, +so a verifier that accepted "could not generate" as a pass would let a host opt +out of the check by declaring a shape the generator does not model. Both a +digest mismatch and an unmodelable shape therefore reject the attestation. The +practical consequence is that CVMs using the TPM key provider (`swtpm = true`) +cannot be verified on either TDX path, which is what the full-image path +already did. + +The guest-side mitigation remains in place as defense in depth. The dangerous +executable part of ACPI is AML (ACPI Machine Language): malicious AML can try to +use `SystemMemory` operation regions through the Linux ACPICA interpreter to +read or write guest physical memory. dstack kernels include the BadAML sandbox +patch (`0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch`), which hooks +the ACPI `SystemMemory` region handler, walks the guest page tables, and denies +AML access to encrypted/private guest RAM. Verification now rejects tampered +tables before the CVM is trusted with keys; the sandbox bounds what tampered +AML could have done in the first place. + +### The kernel measurement does not depend on the host's QEMU + +QEMU is the boot loader for `-kernel`: it fills in the setup-header fields the +Linux boot protocol expects a boot loader to supply, and OVMF measures the +result into RTMR[1]. QEMU commit `a7542a38f399` ("x86/loader: Don't update +kernel header for CoCo VMs", first released in 10.2.0) stopped rewriting the +header for confidential guests, so the same kernel would otherwise measure +differently depending on which QEMU the host chose to run. + +dstack removes that dependency instead of modelling it. The image build zeroes +the boot-loader-written fields in the kernel it ships, and dstack's OVMF zeroes +them again before the kernel blob is measured and loaded. RTMR[1] is therefore +the plain Authenticode hash of the `bzImage` listed in `sha256sum.txt`, and the +verifier needs nothing from the host to predict it -- not a QEMU version, not a +memory size. + +Images built before this landed keep their original behavior: their firmware +does not normalize, so their digest still covers QEMU's rewritten copy. Which +of the two applies is declared by the image itself -- `kernel_header_normalized`, +recorded in `metadata.json` for the image-download path and mirrored into the +measurement document for the no-image-download path -- so it is never something +the host gets to choose. + +Both carriers are bound to `os_image_hash`. `sha256sum.txt` hashes to +`os_image_hash`, a downloaded image is checked file by file against it, and the +measurement document is one of its entries. + +This matters because everything the host declares about its own VM is +untrusted. A knob the verifier has to consult is a knob the host can lie about; +here there is no knob. It also removes a class of correct-but-rejected +deployments, since the previous QEMU-patched digest varied with guest RAM and +was only reproducible at specific memory sizes. + +The normalized field set comes from the boot protocol rather than from QEMU's +behavior: every field `Documentation/arch/x86/boot.rst` types as `write` is one +the boot loader fills in and the kernel supplies no value for, so zeroing it +discards nothing the kernel provided. Fields typed `modify` carry real +kernel-supplied values and are left measured. ### TCB status is surfaced, not gated, during verification @@ -209,19 +368,21 @@ The one case dstack does not leave to downstream is a genuinely invalid TCB: `dc ### Development modes are auditable, not production-safe -dstack keeps several development switches as runtime or on-chain configuration rather than Cargo feature flags. Examples include KMS `quote_enabled = false`, `auth_api.type = "dev"`, and KMS contract `gateway_app_id = "any"`. These settings exist for local development and integration tests, not for production deployments. +dstack keeps several development switches as runtime or on-chain configuration rather than Cargo feature flags. Examples include KMS `attest_rpc_cert = false`, KMS `auth_api.type = "dev"`, and KMS contract `gateway_app_id = "any"`. These settings exist for local development and integration tests, not for production deployments. This is intentional. Runtime configuration that affects the trust boundary is visible in attestation measurements or public contract state. Cargo feature gates are not automatically more auditable because feature unification can enable a feature through a dependency graph, and the resulting runtime behavior is not represented as a measured deployment setting. +This argument has a limit, and it is worth stating because it is what keeps the list short. A switch qualifies only if the trust decision still happens and is merely recorded as a measured setting. A switch that decides *whether* attestation happens at all does not qualify: there is then no measurement to audit, because the thing that would have produced it was skipped. Gateway `core.debug.insecure_skip_attestation` was such a switch -- it turned off both the peer identity check on WaveKV sync and the gateway's own app id lookup -- and it was removed rather than documented. + Production verifiers should reject deployments that use these development settings. Operators should treat them the same way they treat debug-mode TEE quotes: useful for testing, invalid for production trust. ### KMS mTLS is route-enforced for sensitive operations -The KMS Rocket TLS listener permits connections without a client certificate because some bootstrap and public metadata endpoints must be reachable before a client has an RA-TLS certificate. That listener setting is not the authorization boundary for key material. +The KMS Rocket TLS listener permits connections without a client certificate because some bootstrap and public metadata endpoints must be reachable before a client has an RA-TLS certificate. A certificate that is presented must carry an attestation, but the issuer that signed it is not checked and is not the authorization boundary for key material. App key release and KMS key handover require verified caller attestation from the RA-TLS client certificate. Certificate signing verifies the CSR signature and the attestation embedded in the CSR before signing. -The unauthenticated or non-client-certificate surface includes bootstrap and temp-CA bootstrap material retrieval, env-encryption public-key retrieval, metadata, health, and metrics behavior documented for operators. `GetTempCaCert` returns temp CA private material for the bootstrap flow, so operators must treat it as bootstrap-sensitive rather than harmless public metadata. +The unauthenticated or non-client-certificate surface includes bootstrap and temp-CA bootstrap material retrieval, env-encryption public-key retrieval, metadata, health, and metrics behavior documented for operators. `GetTempCaCert` returns temp CA private material and remains in use by guests and by KMS-to-KMS onboarding, which mint their client certificates from that CA; operators must treat it as bootstrap-sensitive rather than harmless public metadata. ## Limitations @@ -233,6 +394,10 @@ Attestation proves which code is running, not that the code is bug-free. It prov Encrypted environment variables prevent the host from reading your secrets. However, the host can replace encrypted values with different ones. Your application should verify authenticity using patterns like LAUNCH_TOKEN. See [security-best-practices.md](./security-best-practices.md) for details. +### `pre_launch_script` is not a launch gate + +`pre_launch_script` runs after dockerd, so a container with a Docker restart policy can be running before it executes — always with `restart: always`, and after any unclean stop with `restart: unless-stopped`. Attestation still binds the script contents, but not the order. Use `init_script`, which runs before dockerd on every boot, for anything that must precede application code. See [security-best-practices.md](./security-best-practices.md#security-semantics-must-not-depend-on-pre_launch_script-running-first) for the failure modes and auditor guidance. + ### KMS root key security All keys derive from the KMS root key, which is protected by TEE isolation. Like all TEE-based systems, a TEE compromise could expose the root key. We are developing MPC-based KMS where the root key is distributed across multiple parties, eliminating this single point of failure. diff --git a/docs/stream-encryption.md b/docs/stream-encryption.md new file mode 100644 index 000000000..3f1a690a3 --- /dev/null +++ b/docs/stream-encryption.md @@ -0,0 +1,77 @@ +# dstack Chunked Encryption Format + +`dstack-util encrypt` and `dstack-util decrypt` use a chunked format for +bounded-memory encryption of arbitrary data. It uses the same app-scoped X25519 +key pair as encrypted environment variables, but it is a separate wire format. + +The header and frame metadata are defined with `binrw`, the fixed-layout binary +codec already used by dstack. Fixed-width integers use little-endian encoding. +The nonce construction below deliberately uses a big-endian chunk index so its +byte representation follows counter order. + +## Header + +| Field | Size | Description | +|---|---:|---| +| Magic | 8 bytes | ASCII `dstkscrt` | +| Version | 1 byte | Format version, currently `0` | +| Ephemeral public key | 32 bytes | X25519 public key generated by the sender | +| Nonce prefix | 8 bytes | Random prefix shared by all chunks | +| Chunk size | 4 bytes | Maximum plaintext bytes in each chunk | + +The X25519 shared secret is used directly as the AES-256-GCM key, matching the +encrypted environment variable protocol. + +## Frames + +Each frame contains: + +| Field | Size | Description | +|---|---:|---| +| Flags | 1 byte | Bit 0 marks the final chunk; all other bits must be zero | +| Plaintext length | 4 bytes | Number of plaintext bytes in this chunk | +| Ciphertext and tag | `plaintext length + 16` bytes | AES-256-GCM output | + +The 12-byte nonce is `nonce_prefix || chunk_index`, where `chunk_index` is a +4-byte integer starting at zero. The authenticated additional data is: + +```text +header || chunk_index || flags || plaintext_length +``` + +Every non-final frame must contain exactly `chunk_size` plaintext bytes. The +final frame may be shorter or empty. A final frame is always emitted, including +for empty input and for input whose length is an exact multiple of the chunk +size. Missing final frames, trailing data, unknown flags, and authentication +failures are rejected. + +## CLI + +Encrypt data after retrieving the app public key over verified TLS: + +```bash +dstack-util encrypt \ + --kms-url https://kms.example.com \ + --app-id "$APP_ID" \ + --kms-pubkey "$TRUSTED_KMS_SIGNER_PUBKEY" \ + --input plaintext.bin \ + --output ciphertext.bin +``` + +`--kms-pubkey` is the trusted compressed secp256k1 public key used to verify the +KMS response's timestamped signature. For a KMS using a private CA, also pass +`--root-ca ca.pem`. Decrypt inside the CVM: + +```bash +dstack-util decrypt --input ciphertext.bin --output plaintext.bin +``` + +`decrypt` detects the magic string automatically. Inputs without the magic are +handled as the legacy encrypted-environment format. Hex input remains available +through `--hex`, but it is decoded in memory and should not be used for large +files. + +Successfully authenticated chunks are written as they are processed. If a +later chunk is corrupt or the final frame is missing, stdout or a file may +therefore contain an authenticated but incomplete plaintext prefix. Callers +must check the command's exit status and discard all output on failure. diff --git a/docs/testing-dstackup-image-pull.md b/docs/testing-dstackup-image-pull.md new file mode 100644 index 000000000..85186967c --- /dev/null +++ b/docs/testing-dstackup-image-pull.md @@ -0,0 +1,39 @@ +# Test `dstackup image pull` with a local release API + +`--release-api-base-url` accepts plain HTTP URLs, including localhost. A small +GitHub Releases API overlay is included for local testing. Locally published +releases shadow GitHub; API requests that are not present locally are relayed to +`https://api.github.com`. + +Start it: + +```bash +./tools/mock-github-releases.py --port 8000 +``` + +Publish a release and copy a local tarball into the mock asset store. The mock +calculates and publishes its SHA-256 digest automatically: + +```bash +curl -fsS -X POST \ + http://localhost:8000/__admin/repos/Dstack-TEE/dstack/releases \ + -H 'content-type: application/json' \ + -d '{ + "tag_name": "guest-os-v99.0.0", + "assets": [{"local_path": "/tmp/dstack-99.0.0.tar.gz"}] + }' +``` + +Pull the locally published latest release: + +```bash +sudo dstackup image pull \ + --release-api-base-url http://localhost:8000/repos \ + --force +``` + +A release can instead reference an already hosted asset by supplying `name`, +`browser_download_url`, and optionally `digest` in the asset object. State is +in memory and resets when the mock exits; copied assets remain in `--asset-dir`. +GitHub API rate limits still apply to relayed requests. Set an `Authorization` +header on a direct request to the mock when testing authenticated relays. diff --git a/docs/tutorials/attestation-verification.md b/docs/tutorials/attestation-verification.md index e39ab8ad0..9808ef94c 100644 --- a/docs/tutorials/attestation-verification.md +++ b/docs/tutorials/attestation-verification.md @@ -129,6 +129,7 @@ Verify you have a running CVM: ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 lsvm ``` @@ -141,6 +142,7 @@ The VMM provides a `/guest/Info` endpoint that proxies into the CVM and retrieve ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) # Get the VM UUID for hello-world @@ -244,46 +246,9 @@ To verify attestation, you need to independently calculate what the measurements cat /var/lib/dstack/images/dstack-0.5.7/metadata.json | jq . ``` -### Build dstack-acpi-tables (required dependency) +### ACPI generation -`dstack-mr` internally runs a tool called `dstack-acpi-tables` to generate ACPI tables for RTMR0 calculation. This is a custom-patched QEMU binary compiled with `-DDUMP_ACPI_TABLES`. You need to build it once: - -```bash -# Install QEMU build dependencies -sudo apt-get update -sudo apt-get install -y git libslirp-dev python3-pip ninja-build \ - pkg-config libglib2.0-dev build-essential flex bison - -# Clone the custom QEMU fork -cd ~/dstack -git clone https://github.com/kvinwang/qemu-tdx.git --depth 1 \ - --branch dstack-qemu-9.2.1 --single-branch - -# Configure with ACPI table dumping enabled -cd qemu-tdx -export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) -export CFLAGS="-DDUMP_ACPI_TABLES -Wno-builtin-macro-redefined -D__DATE__=\"\" -D__TIME__=\"\" -D__TIMESTAMP__=\"\"" -export LDFLAGS="-Wl,--build-id=none" -mkdir build && cd build -../configure --target-list=x86_64-softmmu --disable-werror - -# Build (this takes several minutes) -ninja - -# Install the binary -strip qemu-system-x86_64 -sudo install -m 755 qemu-system-x86_64 /usr/local/bin/dstack-acpi-tables - -# Install required QEMU data files -sudo install -d /usr/local/share/qemu -sudo install -m 644 ../pc-bios/efi-virtio.rom /usr/local/share/qemu/ -sudo install -m 644 ../pc-bios/kvmvapic.bin /usr/local/share/qemu/ -sudo install -m 644 ../pc-bios/linuxboot_dma.bin /usr/local/share/qemu/ - -# Clean up source (optional) -cd ~/dstack -rm -rf qemu-tdx -``` +`dstack-mr` generates QEMU-compatible ACPI measurement data in process. No custom QEMU binary or runtime helper is required. ### Build the measurement calculator @@ -384,6 +349,7 @@ Compare the CVM's actual measurements against your expected values: # verify-measurements.sh cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) # Get VM UUID @@ -518,14 +484,34 @@ These are the standard events you'll see in the log: | `system-preparing` | System initialization marker | Always present | | `app-id` | Application identifier | Should match your app name | | `compose-hash` | SHA-256 of docker compose config | Should match `tcb_info.compose_hash` | +| `init-script-hash` | SHA-256 of one init script; repeated in configured order (maximum 5) | Should match the independently approved script bytes | +| `gpu-policy-hash` | SHA-256 of the JCS-canonicalized GPU policy (default `{}`) | Should match the expected `requirements.gpu_policy` digest | +| `gpu-attestation` | Verified GPU state and digest of the boot-time `nvattest` JSON | Required for an attested GPU launch; verify as described below | | `instance-id` | Unique instance identifier | Should match `instance_id` from response | | `boot-mr-done` | Boot measurements complete | Marker event | -| `mr-kms` | KMS identity measurement | KMS public key hash | | `os-image-hash` | Guest OS image hash | Should match `tcb_info.os_image_hash` | | `key-provider` | Key provider type | e.g., `kms` | | `storage-fs` | Storage filesystem type | Storage configuration | | `system-ready` | System ready marker | Always present at end | +For a successful GPU launch, the relevant order is `compose-hash`, any +`init-script-hash` events, `gpu-policy-hash`, `gpu-attestation`, `instance-id`, +and `boot-mr-done`. +After replaying the log to the quote's RTMR3, decode the JSON payload of +`gpu-attestation` and compare its `evidence_sha256` with the SHA-256 digest of +the boot-time GPU evidence bytes. Fetch those bytes by calling `/v1/Attest` +with `include_boottime_gpu_evidence: true`: `AttestResponse.boottime_gpu_evidence` +is a list of `GpuEvidenceBundle` (`{vendor, format, evidence}`); pick the one +whose `format` is `nvidia-nvattest-boottime-json-v1`, hex-decode its `evidence` +field, and hash exactly those bytes — `SHA-256(hex_decode(bundle.evidence))`. +Hash the decoded bytes as returned, not the JSON string and not a re-serialized +form: the comparison is byte-exact, including any whitespace or trailing +newline. That bundle is the record saved during boot; returning it does not +perform a new attestation. `/v1/AttestGpu` does perform a fresh one, but its +bundles carry the distinct `format` `nvidia-nvattest-collect-evidence-json-v1`, +are appraised against a caller nonce rather than the boot record, and are not +bound to the TD, so they must not be used as remote evidence. + ### Verify specific event values ```bash @@ -567,6 +553,7 @@ echo "Image: $IMAGE_VERSION" echo "" cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) # --- Step 1: Get VM UUID --- @@ -803,19 +790,13 @@ For highest assurance, build images from source: ```bash git clone https://github.com/Dstack-TEE/dstack.git cd dstack -git submodule update --init -- \ - os/yocto/deps/bitbake \ - os/yocto/deps/openembedded-core \ - os/yocto/deps/meta-yocto \ - os/yocto/deps/meta-confidential-compute \ - os/yocto/deps/meta-virtualization \ - os/yocto/deps/meta-openembedded \ - os/yocto/deps/meta-rust-bin \ - os/yocto/deps/meta-security -cd os/yocto/repro-build -./repro-build.sh -n # Reproducible build +git checkout +make os-image # reproducible mkosi build ``` +See [Build the dstack guest OS](../building-guest-os.md) for outputs and the +byte-for-byte reproducibility check. + This ensures you know exactly what code is in the image, and anyone can independently verify the measurements match. ## Troubleshooting @@ -871,4 +852,4 @@ With the foundation complete, you're ready to explore: - [Intel TDX Documentation](https://www.intel.com/content/www/us/en/developer/tools/trust-domain-extensions/documentation.html) - [DCAP Attestation Guide](https://download.01.org/intel-sgx/latest/dcap-latest/linux/docs/) - [dstack Attestation Source](https://github.com/Dstack-TEE/dstack/tree/main/attestation) -- [Reproducible guest-OS builds](../../os/yocto/repro-build/) +- [Reproducible guest-OS builds](../building-guest-os.md) diff --git a/docs/tutorials/clone-build-dstack-vmm.md b/docs/tutorials/clone-build-dstack-vmm.md index 33cc09af0..21bd7b08f 100644 --- a/docs/tutorials/clone-build-dstack-vmm.md +++ b/docs/tutorials/clone-build-dstack-vmm.md @@ -56,13 +56,13 @@ All build commands should be run as the `ubuntu` user. Only the final installati ### Step 2: Verify dstack Repository The dstack repository should already be cloned and checked out on the current -`master` branch from [Local Key Provider](/tutorial/gramine-key-provider): +`next` branch from [Local Key Provider](/tutorial/gramine-key-provider): ```bash cd ~/dstack git describe --tags git branch --show-current -# Should show master +# Should show next ``` ### Step 3: Build dstack-vmm @@ -111,8 +111,8 @@ ls -la /usr/local/bin/dstack-supervisor # Check out a monorepo-era release tag when one is available git checkout -# Or use the master branch for latest development -git checkout master +# Or use the next branch for latest development +git checkout next git pull --ff-only ``` diff --git a/docs/tutorials/contract-deployment.md b/docs/tutorials/contract-deployment.md index e01085edb..b5e429871 100644 --- a/docs/tutorials/contract-deployment.md +++ b/docs/tutorials/contract-deployment.md @@ -49,12 +49,12 @@ These contracts use the UUPS (Universal Upgradeable Proxy Standard) pattern for ### Step 1: Clone Repository and Navigate to auth-eth -On your **local machine**, clone the dstack repository (if you haven't already) and use the current `master` branch: +On your **local machine**, clone the dstack repository (if you haven't already) and use the current `next` branch: ```bash git clone https://github.com/Dstack-TEE/dstack.git ~/dstack 2>/dev/null || true cd ~/dstack -git checkout master +git checkout next cd dstack/kms/auth-eth ``` diff --git a/docs/tutorials/gateway-build-configuration.md b/docs/tutorials/gateway-build-configuration.md index 3a807f8df..8964385c9 100644 --- a/docs/tutorials/gateway-build-configuration.md +++ b/docs/tutorials/gateway-build-configuration.md @@ -460,6 +460,7 @@ Now generate the VMM deployment manifest: ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 compose \ diff --git a/docs/tutorials/gateway-service-setup.md b/docs/tutorials/gateway-service-setup.md index ba693486e..10b592311 100644 --- a/docs/tutorials/gateway-service-setup.md +++ b/docs/tutorials/gateway-service-setup.md @@ -85,6 +85,7 @@ cd ~/gateway-deploy set -a; source .env; set +a cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 deploy \ @@ -318,6 +319,20 @@ curl -sf -X POST "http://$ADMIN_ADDR/prpc/SetCertbotConfig" \ }' && echo "Certbot config set (PRODUCTION)" ``` +The stored ACME account still belongs to the staging directory, so rotate the +shared credentials. This registers a production account and re-pins every ZT +domain's CAA records to it in one step (renewals refuse to run while the +stored account and the configured ACME URL disagree): + +```bash +curl -sf -X POST "http://$ADMIN_ADDR/prpc/RotateAcmeCredentials" \ + -H "Content-Type: application/json" -d '{}' && echo "ACME account rotated" +``` + +> If the rotation reports that CAA re-pinning failed for some domains, the new +> account is already published — rerun `SetCaa` until it succeeds instead of +> rotating again (each rotation registers a new rate-limited ACME account). + After switching the ACME URL, the renewal loop may report "does not need renewal" because the staging cert is still valid. Force a renewal for each ZT domain to get production certificates immediately: ```bash @@ -408,6 +423,7 @@ Navigate to the VMM directory first: ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ``` diff --git a/docs/tutorials/gramine-key-provider.md b/docs/tutorials/gramine-key-provider.md index 1a73306d4..b819e7c41 100644 --- a/docs/tutorials/gramine-key-provider.md +++ b/docs/tutorials/gramine-key-provider.md @@ -70,10 +70,10 @@ git clone https://github.com/Dstack-TEE/dstack.git cd dstack/dstack/local-key-provider/build ``` -The bundled `sgx_default_qcnl.conf` points AESM at the Phala PCCS. To use a -different PCCS, edit its `pccs_url` while keeping certificate verification -enabled, and export the provider's base URL as `PCCS_URL` before running -Compose. +Compose mounts the host's `/etc/sgx_default_qcnl.conf` into the AESM +container. Make sure that file points at a working PCCS before starting the +services. To use a different PCCS for the provider, export its base URL as +`PCCS_URL` before running Compose. Build and start both AESM and the provider: diff --git a/docs/tutorials/guest-image-setup.md b/docs/tutorials/guest-image-setup.md index 555b1ef1f..98f4f617d 100644 --- a/docs/tutorials/guest-image-setup.md +++ b/docs/tutorials/guest-image-setup.md @@ -24,7 +24,7 @@ This tutorial guides you through setting up guest OS images for deploying Confid ## What You'll Configure -- **Guest OS images** - Pre-built Yocto-based images for CVMs +- **Guest OS images** - Pre-built guest OS images for CVMs - **VMM image directory** - Proper organization for multiple image versions - **Image verification** - Confirm VMM can access the images @@ -56,7 +56,8 @@ If you prefer to set up guest images manually, follow these steps. To produce the image from source instead of downloading a release, follow the [guest-OS build guide](../building-guest-os.md). Use the generated -`os/yocto/repro-build/dist/dstack-.tar.gz` archive in Step 3 below. +`os/mkosi/repro-build/build/out/prod/dstack-.tar.gz` archive in Step 3 +below. ### Step 1: Create Image Directory Structure @@ -396,5 +397,5 @@ With guest images configured and VMM able to access them, you're ready to deploy - [Guest OS source and build backends](../../os/) - [dstack GitHub Repository](https://github.com/Dstack-TEE/dstack) -- [Yocto Project](https://www.yoctoproject.org/) +- [mkosi](https://github.com/systemd/mkosi) - [TDX Guest Architecture](https://www.intel.com/content/www/us/en/developer/tools/trust-domain-extensions/overview.html) diff --git a/docs/tutorials/hello-world-app.md b/docs/tutorials/hello-world-app.md index 2792e5ac7..51d1ce656 100644 --- a/docs/tutorials/hello-world-app.md +++ b/docs/tutorials/hello-world-app.md @@ -184,6 +184,7 @@ Use `vmm-cli.py compose` to generate the encrypted deployment manifest. The `--g ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 compose \ @@ -265,6 +266,7 @@ ssh user@your-server ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) SRV_DOMAIN=$(grep ^SRV_DOMAIN ~/gateway-deploy/.env | cut -d= -f2) @@ -373,6 +375,7 @@ Navigate to the VMM directory: ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ``` diff --git a/docs/tutorials/kms-build-configuration.md b/docs/tutorials/kms-build-configuration.md index c4ef875b2..f8264e3a2 100644 --- a/docs/tutorials/kms-build-configuration.md +++ b/docs/tutorials/kms-build-configuration.md @@ -183,14 +183,11 @@ port = 9100 key = "/etc/kms/certs/rpc.key" certs = "/etc/kms/certs/rpc.crt" -# Mutual TLS (mTLS) Configuration -[rpc.tls.mutual] -ca_certs = "/etc/kms/certs/tmp-ca.crt" -# Keep the TLS listener optional because bootstrap/public endpoints must be -# reachable before a client has an RA-TLS certificate. Temp-CA bootstrap material -# is bootstrap-sensitive. Key-release RPCs still require verified caller +# No mutual-TLS section: client certificates are verified by the attestation they +# carry, not by an issuer CA, so there is nothing to pin. Connections without a +# certificate are still accepted, because bootstrap and public endpoints must be +# reachable before a client has one. Key-release RPCs still require verified caller # attestation; certificate signing verifies CSR signature and attestation. -mandatory = false # Core KMS Configuration [core] @@ -371,16 +368,11 @@ The Dockerfile bakes all configuration into the image for reliable CVM deploymen ```bash cat > Dockerfile << 'EOF' # KMS Docker Image for CVM Deployment -# Extract dstack-acpi-tables and QEMU BIOS files from the official builder image. -# These are required for OS image verification (computing expected TDX measurements). -FROM dstacktee/dstack-kms@sha256:11ac59f524a22462ccd2152219b0bec48a28ceb734e32500152d4abefab7a62a AS official - FROM ubuntu:24.04 # Install runtime dependencies -# libglib2.0-0t64, libpixman-1-0, and libslirp0 are required by dstack-acpi-tables (QEMU binary) RUN apt-get update && \ - apt-get install -y ca-certificates curl libglib2.0-0t64 libpixman-1-0 libslirp0 && \ + apt-get install -y ca-certificates curl && \ rm -rf /var/lib/apt/lists/* # Install Node.js 20.x for auth-eth @@ -391,10 +383,6 @@ RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \ # Create directories RUN mkdir -p /etc/kms/certs /etc/kms/images /var/run/kms /var/log/kms -# Copy dstack-acpi-tables from official image (needed for OS image verification) -COPY --from=official /usr/local/bin/dstack-acpi-tables /usr/local/bin/dstack-acpi-tables -COPY --from=official /usr/local/share/qemu /usr/local/share/qemu - # Copy KMS binary COPY dstack-kms /usr/local/bin/dstack-kms RUN chmod 755 /usr/local/bin/dstack-kms @@ -466,14 +454,11 @@ port = 9100 key = "/etc/kms/certs/rpc.key" certs = "/etc/kms/certs/rpc.crt" -# Mutual TLS (mTLS) Configuration -[rpc.tls.mutual] -ca_certs = "/etc/kms/certs/tmp-ca.crt" -# Keep the TLS listener optional because bootstrap/public endpoints must be -# reachable before a client has an RA-TLS certificate. Temp-CA bootstrap material -# is bootstrap-sensitive. Key-release RPCs still require verified caller +# No mutual-TLS section: client certificates are verified by the attestation they +# carry, not by an issuer CA, so there is nothing to pin. Connections without a +# certificate are still accepted, because bootstrap and public endpoints must be +# reachable before a client has one. Key-release RPCs still require verified caller # attestation; certificate signing verifies CSR signature and attestation. -mandatory = false # Core KMS Configuration [core] diff --git a/docs/tutorials/kms-cvm-deployment.md b/docs/tutorials/kms-cvm-deployment.md index bf34ad6cd..2c3cb278b 100644 --- a/docs/tutorials/kms-cvm-deployment.md +++ b/docs/tutorials/kms-cvm-deployment.md @@ -73,8 +73,8 @@ Inside the CVM at `/etc/kms/certs/`: | `root-ca.key` | Root CA signing key (P256 ECDSA) | | `rpc.crt` | TLS certificate for RPC server | | `rpc.key` | RPC server private key | -| `tmp-ca.crt` | Temporary CA for mutual TLS | -| `tmp-ca.key` | Temporary CA private key | +| `tmp-ca.crt` | Temp CA, served by the legacy `GetTempCaCert` bootstrap RPC | +| `tmp-ca.key` | Temp CA private key, served alongside it | | `root-k256.key` | Ethereum signing key (secp256k1) | | `bootstrap-info.json` | Public keys and TDX attestation quote | @@ -165,13 +165,11 @@ configs: key = "/etc/kms/certs/rpc.key" certs = "/etc/kms/certs/rpc.crt" - [rpc.tls.mutual] - ca_certs = "/etc/kms/certs/tmp-ca.crt" - # Keep the TLS listener optional because bootstrap/public endpoints must be - # reachable before a client has an RA-TLS certificate. Temp-CA bootstrap - # material is bootstrap-sensitive. Key-release RPCs still require verified - # caller attestation; certificate signing verifies CSR signature and attestation. - mandatory = false + # No mutual-TLS section: client certificates are verified by the attestation + # they carry, not by an issuer CA. Connections without a certificate are still + # accepted, because bootstrap and public endpoints must be reachable before a + # client has one. Key-release RPCs still require verified caller attestation; + # certificate signing verifies CSR signature and attestation. [core] cert_dir = "/etc/kms/certs" @@ -256,6 +254,7 @@ Use the VMM CLI tool to deploy the CVM: cd ~/dstack/dstack/vmm # Set VMM auth from saved token +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) # Generate app-compose.json with local key provider enabled @@ -306,7 +305,7 @@ curl -s -H "Authorization: Bearer $(cat ~/.dstack/secrets/vmm-auth-token)" \ "http://127.0.0.1:9080/logs?id=VM_ID&follow=true&ansi=false" ``` -> **Note:** The VMM logs endpoint requires Bearer token authentication. The `vmm-cli.py logs` command may not work with token auth — use curl directly as shown above. +> **Note:** The VMM logs endpoint requires authentication. `vmm-cli.py logs` sends credentials automatically when `DSTACK_VMM_TOKEN` (or `--token`) is set, so it works against an auth-enabled VMM; the curl form above is an equivalent alternative. Look for these log messages indicating KMS entered onboard mode: ``` diff --git a/docs/tutorials/troubleshooting-first-application.md b/docs/tutorials/troubleshooting-first-application.md index ac5c11647..5738ab06e 100644 --- a/docs/tutorials/troubleshooting-first-application.md +++ b/docs/tutorials/troubleshooting-first-application.md @@ -84,6 +84,7 @@ If `/guest/Info` returns empty or errors, check that the CVM is running: ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 lsvm ``` diff --git a/docs/tutorials/troubleshooting-gateway-deployment.md b/docs/tutorials/troubleshooting-gateway-deployment.md index 1ef5846d1..650b30ba8 100644 --- a/docs/tutorials/troubleshooting-gateway-deployment.md +++ b/docs/tutorials/troubleshooting-gateway-deployment.md @@ -59,6 +59,7 @@ sudo systemctl restart dstack-vmm **"Authentication required"** — Set the auth token: ```bash +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ``` @@ -72,6 +73,7 @@ export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) # Get KMS VM ID and remove it diff --git a/docs/tutorials/troubleshooting-kms-deployment.md b/docs/tutorials/troubleshooting-kms-deployment.md index 7a4219ea0..7282db94d 100644 --- a/docs/tutorials/troubleshooting-kms-deployment.md +++ b/docs/tutorials/troubleshooting-kms-deployment.md @@ -231,6 +231,7 @@ cat ~/kms-deployment/docker-compose.yml | grep ports -A2 # Check CVM status via vmm-cli.py cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 lsvm ``` diff --git a/docs/tutorials/troubleshooting-prerequisites.md b/docs/tutorials/troubleshooting-prerequisites.md index 0cf57a7ac..64449eab5 100644 --- a/docs/tutorials/troubleshooting-prerequisites.md +++ b/docs/tutorials/troubleshooting-prerequisites.md @@ -289,7 +289,7 @@ docker logs aesmd 2>&1 | tail -30 **Solution:** 1. Verify QCNL configuration points to `https://pccs.phala.network/sgx/certification/v4/` 2. Check network connectivity: `curl -sk https://pccs.phala.network/sgx/certification/v4/rootcacrl` -3. Verify the QCNL config file exists at `~/dstack/dstack/local-key-provider/build/sgx_default_qcnl.conf` +3. Verify the host QCNL config file exists at `/etc/sgx_default_qcnl.conf` ### HTTP health checks fail diff --git a/docs/tutorials/vmm-configuration.md b/docs/tutorials/vmm-configuration.md index 4faf40cbc..07e06426a 100644 --- a/docs/tutorials/vmm-configuration.md +++ b/docs/tutorials/vmm-configuration.md @@ -105,13 +105,14 @@ qemu_path = "" kms_urls = ["http://127.0.0.1:8081"] gateway_urls = ["http://127.0.0.1:8082"] pccs_url = "https://pccs.phala.network/sgx/certification/v4" +# Optional. See "NVIDIA GPU attestation cache" below. +# nvidia_attestation_proxy_url = "http://10.0.2.2:8090" docker_registry = "" cid_start = 1000 cid_pool_size = 1000 max_allocable_vcpu = 124 # Adjust: total cores - 4 max_allocable_memory_in_mb = 990616 # Adjust: total MB - 16384 qmp_socket = false -user = "" use_mrconfigid = true qemu_pci_hole64_size = 0 qemu_hotplug_off = false @@ -212,6 +213,24 @@ Verify QGS is running: systemctl status qgsd ``` +### Optional: NVIDIA GPU attestation cache + +GPU images perform local NVIDIA attestation before app keys are provisioned. +Without a cache this contacts NVIDIA's OCSP and RIM services during every cold +boot. A fleet can run the persistent +[`dstack-nvidia-attest-proxy`](../../dstack/nvidia-attest-proxy/README.md) and pass its +URL to guests through sys-config: + +```toml +[cvm] +nvidia_attestation_proxy_url = "http://10.0.2.2:8090" +``` + +The example address is the host as seen from QEMU user-mode networking. The +proxy must be reachable during `dstack-prepare`. It stores only NVIDIA-signed +collateral and never becomes a signing trust anchor. OCSP entries are not +served after their signed validity window. + ### Step 7: Create Runtime Directories ```bash @@ -286,10 +305,16 @@ To enable GPU passthrough for AI/ML workloads: ```toml [cvm.gpu] enabled = true -listing = ["10de:2335"] # NVIDIA GPU product IDs +# Example: narrow this list to what the host actually holds if you want discovery to reject anything else. +listing = ["10de:2335", "10de:3182"] # H200 SXM, B300 SXM6 allow_attach_all = true ``` +A card whose product ID is not in `listing` is never offered for passthrough, +so the shipped default names every Hopper and Blackwell SKU dstack runs on — +see `[cvm.gpu]` in `dstack/vmm/vmm.toml` for the annotated list. NVSwitches are +not listed: the `all` attach mode finds GPUs and switches by PCI class instead. + **Requirements:** - IOMMU enabled in BIOS - VFIO driver configured diff --git a/docs/usage.md b/docs/usage.md index 08f072f0d..85702f2c9 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -82,6 +82,18 @@ services: curl --unix-socket /var/run/dstack.sock http://localhost/GetQuote?report_data=0x1234deadbeef | jq . ``` +`GetQuote` needs Intel TDX. On platforms without it (AMD SEV-SNP, AWS Nitro) it +returns an error. On GCP Confidential VMs it answers, but with the TDX quote +alone -- GCP's verification also binds a vTPM quote, which this response has no +field for, so a verifier that takes it at face value checks less than it should. + +`Attest` is the replacement in both cases. It returns a platform-adaptive +attestation carrying whatever evidence the platform actually produces: + +```bash +curl --unix-socket /var/run/dstack.sock http://localhost/Attest?report_data=0x1234deadbeef | jq . +``` + For advanced compatibility with unmodified binaries that expect native Linux TEE interfaces such as `/dev/tdx_guest`, `/dev/sev-guest`, or configfs-tsm, see [Advanced Native TEE Interfaces in Containers](./native-tee-interfaces.md). ## Container Logs diff --git a/docs/verification.md b/docs/verification.md index 1f54857a4..7b49fe36f 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -18,7 +18,7 @@ If any of these fail, the cryptographic proof won't verify. **Programmatic verification**: dstack provides several tools: -- [dstack-verifier](https://github.com/Dstack-TEE/dstack/tree/master/dstack/verifier) - HTTP service with `/verify` endpoint, also runs as CLI +- [dstack-verifier](https://github.com/Dstack-TEE/dstack/tree/next/dstack/verifier) - HTTP service with `/verify` endpoint, also runs as CLI - [dcap-qvl](https://github.com/Phala-Network/dcap-qvl) - Open source quote verification library (Rust, Python, JS/WASM, CLI) - [SDKs](../sdk/) - JavaScript and Python SDKs include `replayRtmrs()` for local RTMR verification diff --git a/docs/verity-volumes.md b/docs/verity-volumes.md new file mode 100644 index 000000000..6c7dac295 --- /dev/null +++ b/docs/verity-volumes.md @@ -0,0 +1,89 @@ +# Verity data volumes + +A verity volume is a read-only filesystem image protected by dm-verity. The host +attaches untrusted bytes; the guest mounts them only when they match the root +hash measured in `app-compose.json`. + +## Disk format + +A built volume is a raw GPT image: + +```text +p1: DSTACK_VOLUME metadata envelope +p2: filesystem data +p3: dm-verity superblock and hash tree +``` + +The envelope identifies a candidate disk. It is not trusted. The guest passes +the root from the measured app compose to `veritysetup`, so a forged envelope or +partition table cannot substitute different contents. + +## Build + +Pack a directory as a reproducible squashfs volume: + +```bash +dstack verity --dir ./models -o models.img +``` + +Or wrap an existing filesystem image: + +```bash +dstack verity --fs-image ./models.ext4 -o models.img +``` + +The command prints the root and a deploy argument. Place the image in the VMM's +configured `cvm.volumes_dir`, then deploy it by bare file name, root, and guest +mount point: + +```bash +dstack deploy -c docker-compose.yaml \ + --volume models.img:a1b2c3d4...:/run/models +``` + +This adds the following measured entry to `app-compose.json`: + +```json +{ + "verity_volumes": [ + { + "source": "models.img", + "verity_root": "a1b2c3d4...", + "target": "/run/models" + } + ] +} +``` + +The target must be an absolute path on a writable guest filesystem, such as +`/run` or the app data disk. The guest root filesystem itself is read-only. + +## Guest activation + +Before the application starts, `dstack-volume mount-all app-compose.json`: + +1. Scans `/sys/class/block` for disks whose first partition, or whole disk when + unpartitioned, starts with the `DSTACK_VOLUME` magic. +2. Matches the full root advertised by the envelope against the measured root. +3. Opens p2 and p3 with `veritysetup` using the measured root. +4. Reads the first mapped block to force an initial integrity check. +5. Mounts the mapped filesystem read-only at the measured target. + +A required volume that is missing, malformed, or fails verification stops guest +preparation. dm-verity continues verifying blocks lazily as the application +reads them. + +The same `verity_root` may be declared more than once with different targets. +The VMM attaches one disk for that root, and the guest mounts the verified +content at each requested target. + +For diagnostics, `dstack-volume scan` lists recognized disks and +`dstack-volume status app-compose.json` compares requested roots with attached +and active devices. + +## Trust and limitations + +The root hash authenticates bytes, not availability. A malicious host can omit a +volume or cause I/O failures, but cannot silently replace its contents. Volumes +are unencrypted and are intended for public, shareable data; confidential data +requires a separate encryption design. diff --git a/docs/vmm-cli-user-guide.md b/docs/vmm-cli-user-guide.md index bb93315d9..6dddc2ba5 100644 --- a/docs/vmm-cli-user-guide.md +++ b/docs/vmm-cli-user-guide.md @@ -70,26 +70,46 @@ export DSTACK_VMM_URL=unix:/path/to/socket ### Authentication -If your dstack-vmm server requires authentication, you can provide credentials using: +When the dstack-vmm server has `[auth] enabled = true`, the token now guards the +*entire* management surface (listing/creating/stopping VMs, deploys, logs, and +the web UI) — not just the logs endpoint. Provide credentials in one of two +forms. -#### Environment Variables (Recommended) +#### Bearer token (Recommended) -```bash -# Set authentication credentials -export DSTACK_VMM_AUTH_USER=your-username -export DSTACK_VMM_AUTH_PASSWORD=your-password +Pass the VMM API token directly; it is sent as `Authorization: Bearer `. -# Then use CLI normally +```bash +# your VMM API token. `dstackup install` writes it to +# /vmm-auth-token (default /etc/dstack/vmm-auth-token) — and the +# local `dstack` CLI reads it automatically; a manual setup (see the VMM +# configuration tutorial) stores it at ~/.dstack/secrets/vmm-auth-token. +export DSTACK_VMM_TOKEN=$(cat /etc/dstack/vmm-auth-token) ./vmm-cli.py lsvm + +# or as a flag +./vmm-cli.py --token "$DSTACK_VMM_TOKEN" lsvm ``` -#### Command Line Arguments +#### HTTP Basic + +The server also accepts HTTP Basic, where the password may be the shared token +(any username, e.g. `admin`) or an entry in the server's `htpasswd_file`. ```bash -./vmm-cli.py --auth-user your-username --auth-password your-password lsvm +export DSTACK_VMM_AUTH_USER=admin +export DSTACK_VMM_AUTH_PASSWORD=$(cat /etc/dstack/vmm-auth-token) +./vmm-cli.py lsvm + +# or as flags +./vmm-cli.py --auth-user admin --auth-password "$DSTACK_VMM_AUTH_PASSWORD" lsvm ``` -**Note:** Environment variables take precedence over command line arguments for authentication. +**Note:** A bearer token takes precedence over Basic when both are set. For each +setting, command-line flags take precedence over environment variables, which +take precedence over the config file. Setting only one half of a Basic +credential (e.g. `DSTACK_VMM_AUTH_PASSWORD` without `DSTACK_VMM_AUTH_USER`) is +rejected with an error rather than silently sending an unauthenticated request. ## Basic Commands @@ -205,7 +225,7 @@ First, create an application composition file that describes your application: - `--name`: Friendly name for your application - `--docker-compose`: Path to your Docker Compose file -- `--prelaunch-script`: Optional script to run before starting containers +- `--prelaunch-script`: Optional script to run before starting containers. It runs after dockerd, so containers restored by a Docker restart policy may already be running; do not put security gates here (see [security best practices](./security/security-best-practices.md#security-semantics-must-not-depend-on-pre_launch_script-running-first)) - `--kms`: Enable Key Management Service for secrets - `--gateway`: Enable dstack-gateway for external access - `--local-key-provider`: Use local key provider @@ -270,6 +290,11 @@ Expose services running in your VM: # Multiple ports --port tcp:8080:80 --port tcp:8443:443 + +# Pin a mapping to one NIC: protocol[:host_address]:host_port:vm_port@ +# Without @ the mapping enters through the first user-mode NIC. A VM +# with no user-mode NIC has no backend that can publish the mapping. +--port tcp:0.0.0.0:8443:443@0 ``` #### GPU Assignment @@ -325,8 +350,7 @@ After successful deployment, verify your VM is running correctly: export DSTACK_VMM_URL=http://127.0.0.1:12000 # If authentication is required -export DSTACK_VMM_AUTH_USER=your-username -export DSTACK_VMM_AUTH_PASSWORD=your-password +export DSTACK_VMM_TOKEN=$(cat ~/.dstack/secrets/vmm-auth-token) # Create a basic docker-compose.yml cat > docker-compose.yml << 'EOF' diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 9d2f172e7..227ef2fae 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "acpi_tables" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce821f856a3eb1d033287f2dcfcdf94276d7895dd5bdd8ca45ae17e7d33d4dd9" +dependencies = [ + "zerocopy", +] + [[package]] name = "adler2" version = "2.0.1" @@ -55,6 +64,20 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -180,7 +203,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" dependencies = [ - "asn1-rs-derive", + "asn1-rs-derive 0.5.1", "asn1-rs-impl", "displaydoc", "nom", @@ -190,6 +213,22 @@ dependencies = [ "time", ] +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive 0.6.0", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + [[package]] name = "asn1-rs-derive" version = "0.5.1" @@ -202,6 +241,18 @@ dependencies = [ "synstructure", ] +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + [[package]] name = "asn1-rs-impl" version = "0.2.0" @@ -310,6 +361,58 @@ dependencies = [ "serde_cbor", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "base16ct" version = "0.2.0" @@ -343,6 +446,19 @@ dependencies = [ "serde", ] +[[package]] +name = "bcrypt" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abaf6da45c74385272ddf00e1ac074c7d8a6c1a1dda376902bd6a427522a8b2c" +dependencies = [ + "base64 0.22.1", + "blowfish", + "getrandom 0.3.4", + "subtle", + "zeroize", +] + [[package]] name = "binascii" version = "0.1.4" @@ -401,6 +517,30 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec 0.8.0", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitfield" version = "0.15.0" @@ -483,6 +623,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blowfish" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e412e2cd0f2b2d93e02543ceae7917b3c70331573df19ee046bcbc35e45e87d7" +dependencies = [ + "byteorder", + "cipher 0.4.4", +] + [[package]] name = "bollard" version = "0.18.1" @@ -543,7 +693,7 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" dependencies = [ - "darling", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", @@ -552,6 +702,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + [[package]] name = "borsh" version = "1.6.1" @@ -608,6 +764,12 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7575182f7272186991736b70173b0ea045398f984bf5ebbb3804736ce1330c9d" +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytemuck" version = "1.25.0" @@ -651,13 +813,16 @@ version = "0.6.0" dependencies = [ "anyhow", "digest 0.10.7", + "dstack-types", "ez-hash", "fs-err", "hex", "insta", + "or-panic", "parity-scale-codec", "serde", "serde-human-bytes", + "serde_jcs", "serde_json", "sha2 0.10.9", ] @@ -667,12 +832,17 @@ name = "cert-client" version = "0.6.0" dependencies = [ "anyhow", + "dstack-guest-agent-rpc", "dstack-kms-rpc", "dstack-types", + "hex", + "http-client", "ra-rpc", "ra-tls", "serde_json", "tdx-attest", + "tokio", + "x509-parser 0.16.0", ] [[package]] @@ -690,15 +860,16 @@ dependencies = [ "instant-acme", "path-absolutize", "rand 0.8.6", - "rcgen", + "rcgen 0.13.2", "reqwest", "serde", "serde_json", + "tempfile", "time", "tokio", "tracing", "tracing-subscriber", - "x509-parser", + "x509-parser 0.16.0", ] [[package]] @@ -755,6 +926,16 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + [[package]] name = "ciborium" version = "0.2.2" @@ -1031,6 +1212,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -1151,7 +1347,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", - "x509-parser", + "x509-parser 0.16.0", ] [[package]] @@ -1163,17 +1359,6 @@ dependencies = [ "cipher 0.4.4", ] -[[package]] -name = "cuckoofilter" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b810a8449931679f64cd7eef1bbd0fa315801b6d5d9cdc1ace2804d6529eee18" -dependencies = [ - "byteorder", - "fnv", - "rand 0.7.3", -] - [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -1201,14 +1386,38 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", ] [[package]] @@ -1224,13 +1433,24 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.117", +] + [[package]] name = "darling_macro" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core", + "darling_core 0.23.0", "quote", "syn 2.0.117", ] @@ -1257,9 +1477,9 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "dcap-qvl" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92a14fb8954c867d6855e44d98eab18e769816357738406691ebe60d8fdd005d" +checksum = "8b3d93e370c7f1bc394106d8f1dc7b58d7d23e765c95b084fe3657efe02ba166" dependencies = [ "anyhow", "asn1_der", @@ -1346,7 +1566,21 @@ version = "9.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" dependencies = [ - "asn1-rs", + "asn1-rs 0.6.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs 0.7.2", "displaydoc", "nom", "num-bigint", @@ -1570,6 +1804,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dstack-api-auth" +version = "0.6.0" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bcrypt", + "rocket", + "sha2 0.10.9", + "subtle", +] + [[package]] name = "dstack-attest" version = "0.6.0" @@ -1593,9 +1839,13 @@ dependencies = [ "nsm-qvl", "or-panic", "parity-scale-codec", + "pem", "rand 0.8.6", + "rcgen 0.13.2", "rmp-serde", "rsa", + "rustix 0.38.44", + "safe-write", "serde", "serde-human-bytes", "serde_json", @@ -1604,12 +1854,14 @@ dependencies = [ "sha2 0.10.9", "sha3", "tdx-attest", + "tempfile", "tokio", "tpm-attest", "tpm-qvl", "tpm-types", "tpm2", "tracing", + "x509-parser 0.16.0", ] [[package]] @@ -1623,6 +1875,13 @@ dependencies = [ "serde_json", ] +[[package]] +name = "dstack-build-info" +version = "0.6.0" +dependencies = [ + "git-version", +] + [[package]] name = "dstack-cli" version = "0.6.0" @@ -1630,8 +1889,14 @@ dependencies = [ "anyhow", "clap", "dstack-cli-core", + "dstack-types", + "dstack-volume", + "fs-err", + "hex", "serde_json", "tokio", + "tracing", + "tracing-subscriber", ] [[package]] @@ -1639,9 +1904,11 @@ name = "dstack-cli-core" version = "0.6.0" dependencies = [ "anyhow", + "dstack-types", "dstack-vmm-rpc", "http-client", "rustix 0.38.44", + "safe-write", "serde_json", "toml", ] @@ -1658,14 +1925,15 @@ dependencies = [ "certbot", "clap", "cmd_lib", + "dstack-api-auth", + "dstack-attest", + "dstack-build-info", "dstack-gateway-rpc", "dstack-guest-agent-rpc", - "dstack-kms-rpc", "dstack-types", "flate2", "fs-err", "futures", - "git-version", "hex", "hickory-resolver", "http-body-util", @@ -1676,6 +1944,8 @@ dependencies = [ "insta", "ipnet", "jemallocator", + "ktls", + "libc", "load_config", "nix 0.29.0", "or-panic", @@ -1687,7 +1957,9 @@ dependencies = [ "rand 0.8.6", "reqwest", "rinja", + "rmp", "rmp-serde", + "rmpv", "rocket", "rustls", "safe-write", @@ -1697,7 +1969,7 @@ dependencies = [ "sha2 0.10.9", "shared_child", "smallvec", - "subtle", + "socket2 0.5.10", "tdx-attest", "tempfile", "tokio", @@ -1705,8 +1977,9 @@ dependencies = [ "tracing", "tracing-subscriber", "uuid", - "wavekv", - "x509-parser", + "wavekv 1.0.0", + "wavekv 2.1.0", + "x509-parser 0.16.0", ] [[package]] @@ -1729,6 +2002,7 @@ dependencies = [ "anyhow", "base64 0.22.1", "bollard", + "cached-cell", "cc-eventlog", "cert-client", "chrono", @@ -1736,23 +2010,27 @@ dependencies = [ "cmd_lib", "default-net", "dstack-attest", + "dstack-build-info", "dstack-guest-agent-rpc", "dstack-types", "ed25519-dalek", "figment", "fs-err", - "git-version", + "futures", "guest-api", "hex", "host-api", "k256", + "libc", "listenfd", "load_config", + "lspci", + "nvattest", "or-panic", "ra-rpc", "ra-tls", "rand 0.8.6", - "rcgen", + "rcgen 0.13.2", "reqwest", "ring", "rinja", @@ -1778,12 +2056,15 @@ name = "dstack-guest-agent-rpc" version = "0.6.0" dependencies = [ "anyhow", + "hex", "parity-scale-codec", "prost 0.13.5", + "prost-types 0.13.5", "prpc", "prpc-build", "serde", "serde_json", + "sha2 0.10.9", ] [[package]] @@ -1791,9 +2072,14 @@ name = "dstack-guest-agent-simulator" version = "0.6.0" dependencies = [ "anyhow", + "cc-eventlog", "clap", + "dcap-qvl", "dstack-guest-agent", "dstack-guest-agent-rpc", + "dstack-types", + "hex", + "mock-attestation", "ra-rpc", "ra-tls", "rocket", @@ -1809,16 +2095,16 @@ version = "0.6.0" dependencies = [ "anyhow", "cc-eventlog", - "chrono", "clap", + "dstack-api-auth", "dstack-attest", + "dstack-build-info", "dstack-guest-agent-rpc", "dstack-kms-rpc", "dstack-mr", "dstack-types", "dstack-verifier", "fs-err", - "git-version", "hex", "hex_fmt", "http-client", @@ -1843,8 +2129,8 @@ dependencies = [ "tracing", "tracing-subscriber", "x25519-dalek", - "x509-parser", - "yasna", + "x509-parser 0.16.0", + "yasna 0.5.2", ] [[package]] @@ -1876,6 +2162,7 @@ dependencies = [ "log", "object", "parity-scale-codec", + "qemu-acpi", "reqwest", "serde", "serde-human-bytes", @@ -1901,14 +2188,43 @@ dependencies = [ ] [[package]] -name = "dstack-port-forward" +name = "dstack-netd-rpc" version = "0.6.0" dependencies = [ "anyhow", - "nix 0.29.0", + "prost 0.13.5", + "prpc", + "prpc-build", + "serde", + "serde_json", +] + +[[package]] +name = "dstack-nvidia-attest-proxy" +version = "0.6.0" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bytes", + "chrono", + "clap", + "dashmap", + "futures", + "hex", + "http", + "http-body-util", + "humantime", + "hyper", + "hyper-util", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", "tokio", - "tokio-util", "tracing", + "tracing-subscriber", + "url", ] [[package]] @@ -1916,13 +2232,34 @@ name = "dstack-tee-simulator" version = "0.6.0" dependencies = [ "anyhow", + "aws-nitro-enclaves-nsm-api", "cc-eventlog", "clap", "dcap-qvl", + "dstack-attest", + "dstack-mr", + "dstack-types", + "fs-err", "fuser", + "hex", "libc", + "libloading", + "mock-attestation", + "nix 0.29.0", + "nsm-qvl", + "pem", + "reqwest", + "safe-write", "sd-notify", + "serde_cbor", + "serde_json", + "sev-snp-qvl", "sha2 0.10.9", + "tempfile", + "tokio", + "tpm-qvl", + "tpm-types", + "tpm2", "tracing", "tracing-subscriber", ] @@ -1939,6 +2276,7 @@ dependencies = [ "serde-human-bytes", "serde_jcs", "serde_json", + "serde_with", "sha2 0.10.9", "sha3", "size-parser", @@ -1951,6 +2289,7 @@ dependencies = [ "aes-gcm", "anyhow", "base64 0.22.1", + "binrw", "bollard", "cc-eventlog", "cert-client", @@ -1966,21 +2305,26 @@ dependencies = [ "ez-hash", "fs-err", "getrandom 0.3.4", + "guest-api", "hex", "hex_fmt", "host-api", "k256", "libc", + "lspci", "luks2", + "nvattest", + "nvml-wrapper", "parity-scale-codec", "ra-rpc", "ra-tls", "rand 0.8.6", "regex", + "regorus", "safe-write", "schnorrkel", "scopeguard", - "semver", + "sd-notify", "serde", "serde-human-bytes", "serde_json", @@ -1996,8 +2340,9 @@ dependencies = [ "tpm2", "tracing", "tracing-subscriber", + "url", "x25519-dalek", - "x509-parser", + "x509-parser 0.16.0", "yaml-rust2", ] @@ -2014,23 +2359,27 @@ dependencies = [ "dstack-types", "ez-hash", "figment", + "flate2", "fs-err", "hex", "hex-literal", "nsm-attest", "ra-tls", + "rcgen 0.13.2", "reqwest", "rocket", "serde", "serde-human-bytes", "serde_json", "sha2 0.10.9", + "tar", "tempfile", "tokio", "tpm-qvl", "tpm-types", "tracing", "tracing-subscriber", + "x509-parser 0.16.0", ] [[package]] @@ -2042,9 +2391,11 @@ dependencies = [ "bon", "clap", "dirs 6.0.0", + "dstack-api-auth", + "dstack-build-info", "dstack-kms-rpc", "dstack-mr", - "dstack-port-forward", + "dstack-netd-rpc", "dstack-types", "dstack-vmm-rpc", "fatfs", @@ -2052,23 +2403,26 @@ dependencies = [ "fs-err", "fscommon", "getrandom 0.3.4", - "git-version", "guest-api", "hex", "hex_fmt", "host-api", + "http-client", "humantime", "insta", "key-provider-client", + "libc", + "listenfd", "load_config", "lspci", + "mock-attestation", "nix 0.29.0", "or-panic", "path-absolutize", "ra-rpc", + "rand 0.8.6", "reqwest", "rocket", - "rocket-apitoken", "rocket-vsock-listener", "safe-write", "serde", @@ -2082,11 +2436,13 @@ dependencies = [ "supervisor-client", "tailf", "tar", + "tempfile", "tokio", "tracing", "tracing-subscriber", "url", "uuid", + "wait-timeout", "which 7.0.3", ] @@ -2103,6 +2459,27 @@ dependencies = [ "serde_json", ] +[[package]] +name = "dstack-volume" +version = "0.6.0" +dependencies = [ + "anyhow", + "binrw", + "clap", + "cmd_lib", + "dstack-types", + "fs-err", + "gpt", + "hex", + "serde_json", + "sha2 0.10.9", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "uuid", +] + [[package]] name = "dstackup" version = "0.6.0" @@ -2111,6 +2488,7 @@ dependencies = [ "clap", "dstack-cli-core", "hex", + "rand 0.8.6", "reqwest", "serde", "serde_json", @@ -2197,6 +2575,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + [[package]] name = "encode_unicode" version = "1.0.0" @@ -2307,6 +2694,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "fancy-regex" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.4.1" @@ -2400,6 +2798,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "fnv" version = "1.0.7" @@ -2427,6 +2836,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fraction" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "fs-err" version = "3.3.0" @@ -2593,17 +3012,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.17" @@ -2613,7 +3021,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] @@ -2638,11 +3046,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -2691,6 +3101,30 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "globset" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" +dependencies = [ + "aho-corasick", + "bstr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "gpt" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3696fafb1ecdcc2ae3ce337de73e9202806068594b77d22fdf2f3573c5ec2219" +dependencies = [ + "bitflags 2.11.1", + "crc", + "simple-bytes", + "uuid", +] + [[package]] name = "group" version = "0.13.0" @@ -2765,12 +3199,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "hash_hasher" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b4b9ebce26001bad2e6366295f64e381c1e9c479109202149b9e15e154973e9" - [[package]] name = "hashbrown" version = "0.12.3" @@ -2792,6 +3220,17 @@ dependencies = [ "foldhash 0.1.5", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -3015,6 +3454,7 @@ dependencies = [ "hyperlocal", "log", "pin-project-lite", + "prost 0.13.5", "prpc", "reqwest", "serde", @@ -3098,6 +3538,7 @@ dependencies = [ "hyper-util", "rustls", "rustls-native-certs", + "rustls-platform-verifier", "tokio", "tokio-rustls", "tower-service", @@ -3120,7 +3561,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.4", "tokio", "tower-service", "tracing", @@ -3372,24 +3813,28 @@ dependencies = [ [[package]] name = "instant-acme" -version = "0.7.2" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37221e690dcc5d0ea7c1f70decda6ae3495e72e8af06bca15e982193ffdf4fc4" +checksum = "9f05ad37c421b962354c358d347d4a6130151df9407978372d3ad7f0c8f71a64" dependencies = [ "async-trait", + "aws-lc-rs", "base64 0.22.1", "bytes", "http", "http-body", "http-body-util", + "httpdate", "hyper", "hyper-rustls", "hyper-util", - "ring", + "rcgen 0.14.9", + "rustls", "rustls-pki-types", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 2.0.18", + "tokio", ] [[package]] @@ -3426,7 +3871,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" dependencies = [ - "socket2", + "socket2 0.6.4", "widestring", "windows-registry", "windows-result 0.4.1", @@ -3583,6 +4028,42 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonschema" +version = "0.46.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a699d3e77675e6aa4bfffe3b907c8b5f7ed3241f9965bffb25475ad4b08d05" +dependencies = [ + "ahash", + "bytecount", + "data-encoding", + "email_address", + "fancy-regex", + "fraction", + "getrandom 0.3.4", + "idna", + "itoa", + "jsonschema-regex", + "num-cmp", + "num-traits", + "percent-encoding", + "referencing", + "regex", + "serde", + "serde_json", + "unicode-general-category", + "uuid-simd", +] + +[[package]] +name = "jsonschema-regex" +version = "0.46.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbd1086b01b9349fd4ef9a07433965af64c8ce8159abe633a189e4ff817bd13" +dependencies = [ + "regex-syntax", +] + [[package]] name = "k256" version = "0.13.4" @@ -3651,13 +4132,38 @@ dependencies = [ "libc", ] +[[package]] +name = "ktls" +version = "6.0.2" +dependencies = [ + "futures-util", + "ktls-sys", + "libc", + "memoffset 0.9.1", + "nix 0.29.0", + "num_enum", + "pin-project-lite", + "rustls", + "smallvec", + "thiserror 1.0.69", + "tokio", + "tokio-rustls", + "tracing", +] + +[[package]] +name = "ktls-sys" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ed84c81d133bc00e291085cff51c15cb07d3cede0aade1f2d82dd33df82d7e7" + [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.8", ] [[package]] @@ -3672,6 +4178,16 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link 0.2.1", +] + [[package]] name = "libm" version = "0.2.16" @@ -3775,6 +4291,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6" + [[package]] name = "lru-slab" version = "0.1.2" @@ -3787,6 +4309,7 @@ version = "0.6.0" dependencies = [ "anyhow", "insta", + "tempfile", ] [[package]] @@ -3820,6 +4343,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "md-5" version = "0.10.6" @@ -3872,6 +4401,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "micromap" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" + [[package]] name = "mime" version = "0.3.17" @@ -3925,7 +4460,7 @@ checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "log", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "windows-sys 0.61.2", ] @@ -3938,6 +4473,40 @@ dependencies = [ "winapi", ] +[[package]] +name = "mock-attestation" +version = "0.6.0" +dependencies = [ + "anyhow", + "axum", + "chrono", + "ciborium", + "clap", + "dcap-qvl", + "dstack-types", + "fs-err", + "hex", + "nsm-qvl", + "p256", + "p384", + "parity-scale-codec", + "rand 0.8.6", + "rcgen 0.13.2", + "reqwest", + "serde", + "serde_json", + "sev", + "sev-snp-qvl", + "sha2 0.10.9", + "tempfile", + "time", + "tokio", + "tpm-qvl", + "tpm-types", + "urlencoding", + "yasna 0.5.2", +] + [[package]] name = "moka" version = "0.12.15" @@ -3955,6 +4524,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "msvc_spectre_libs" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29e871a9861f3664f18b7e04e9301d4edd55090c2dadb4b1c602e26ab32b1f5b" +dependencies = [ + "cc", +] + [[package]] name = "multer" version = "3.1.0" @@ -3969,7 +4547,7 @@ dependencies = [ "log", "memchr", "mime", - "spin", + "spin 0.9.8", "tokio", "tokio-util", "version_check", @@ -4150,7 +4728,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", - "x509-parser", + "x509-parser 0.16.0", ] [[package]] @@ -4180,6 +4758,20 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -4206,6 +4798,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -4238,6 +4845,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ + "num-bigint", "num-integer", "num-traits", ] @@ -4256,10 +4864,67 @@ dependencies = [ name = "num_cpus" version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "nvattest" +version = "0.6.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "nvml-wrapper" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f049ae562349fefb8e837eb15443da1e7c6dcbd8a11f52a228f92220c2e5c85e" +dependencies = [ + "bitflags 2.11.1", + "libloading", + "nvml-wrapper-sys", + "static_assertions", + "thiserror 1.0.69", + "wrapcenum-derive", +] + +[[package]] +name = "nvml-wrapper-sys" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4d594420fcda43b1c2c4bd44d48974aa3c7a9ab2cbf10dc18e35265767bf0b" dependencies = [ - "hermit-abi", - "libc", + "libloading", ] [[package]] @@ -4298,7 +4963,16 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" dependencies = [ - "asn1-rs", + "asn1-rs 0.6.2", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs 0.7.2", ] [[package]] @@ -4362,6 +5036,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "p256" version = "0.13.2" @@ -5016,6 +5696,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "qemu-acpi" +version = "0.1.0" +dependencies = [ + "acpi_tables", + "hex", + "sha2 0.10.9", + "thiserror 2.0.18", +] + [[package]] name = "quinn" version = "0.11.9" @@ -5029,7 +5719,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -5038,15 +5728,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.2", "lru-slab", - "rand 0.9.4", + "rand 0.10.1", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -5067,7 +5758,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.6.4", "tracing", "windows-sys 0.60.2", ] @@ -5106,10 +5797,12 @@ dependencies = [ "reqwest", "rocket", "rocket-vsock-listener", + "rustls", "serde", "serde_json", + "tokio", "tracing", - "x509-parser", + "x509-parser 0.16.0", ] [[package]] @@ -5122,6 +5815,7 @@ dependencies = [ "dcap-qvl", "dstack-attest", "dstack-types", + "ed25519-dalek", "elliptic-curve", "errify", "ez-hash", @@ -5130,11 +5824,12 @@ dependencies = [ "hex", "hex_fmt", "hkdf", + "k256", "or-panic", "p256", "parity-scale-codec", "rand 0.8.6", - "rcgen", + "rcgen 0.13.2", "ring", "rmp-serde", "rustls-pki-types", @@ -5144,12 +5839,13 @@ dependencies = [ "sha2 0.10.9", "sha3", "tdx-attest", + "tempfile", "tokio", "tpm-qvl", "tpm-types", "tracing", - "x509-parser", - "yasna", + "x509-parser 0.16.0", + "yasna 0.5.2", ] [[package]] @@ -5158,19 +5854,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", -] - [[package]] name = "rand" version = "0.8.6" @@ -5203,16 +5886,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", -] - [[package]] name = "rand_chacha" version = "0.3.1" @@ -5233,15 +5906,6 @@ dependencies = [ "rand_core 0.9.5", ] -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", -] - [[package]] name = "rand_core" version = "0.6.4" @@ -5267,12 +5931,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] -name = "rand_hc" -version = "0.2.0" +name = "rand_pcg" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" dependencies = [ - "rand_core 0.5.1", + "rand_core 0.10.1", ] [[package]] @@ -5285,8 +5949,22 @@ dependencies = [ "ring", "rustls-pki-types", "time", - "x509-parser", - "yasna", + "x509-parser 0.16.0", + "yasna 0.5.2", +] + +[[package]] +name = "rcgen" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091e7a8e7d86e6feb87a27ce8e2cba29d49eff9507afeebefab7eeb2ca667fb4" +dependencies = [ + "aws-lc-rs", + "pem", + "rustls-pki-types", + "time", + "x509-parser 0.18.1", + "yasna 0.6.0", ] [[package]] @@ -5355,6 +6033,23 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09c30c54dffee5b40af088d5d50aa3455c91a0127164b51f0215efc4cb28fb3c" +[[package]] +name = "referencing" +version = "0.46.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fbf332a2f81899f6836f22c03da73dae8a664c32e3016b84692c23cddadc95d" +dependencies = [ + "ahash", + "fluent-uri", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "itoa", + "micromap", + "parking_lot 0.12.5", + "percent-encoding", + "serde_json", +] + [[package]] name = "regex" version = "1.12.3" @@ -5384,6 +6079,38 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "regorus" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419a0413adeece71e4d4a64fb75adc359cb807496f0dfd10f429517be908b807" +dependencies = [ + "anyhow", + "chrono", + "chrono-tz", + "data-encoding", + "globset", + "indexmap 2.14.0", + "ipnet", + "jsonschema", + "lazy_static", + "lru", + "msvc_spectre_libs", + "num-bigint", + "num-traits", + "parking_lot 0.12.5", + "rand 0.10.1", + "regex", + "semver", + "serde", + "serde_json", + "serde_yaml", + "spin 0.10.1", + "thiserror 2.0.18", + "url", + "uuid", +] + [[package]] name = "reqwest" version = "0.13.4" @@ -5524,6 +6251,15 @@ dependencies = [ "serde", ] +[[package]] +name = "rmpv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a4e1d4b9b938a26d2996af33229f0ca0956c652c1375067f0b45291c1df8417" +dependencies = [ + "rmp", +] + [[package]] name = "rocket" version = "0.6.0-dev" @@ -5570,18 +6306,10 @@ dependencies = [ "tracing-subscriber", "ubyte", "version_check", - "x509-parser", + "x509-parser 0.16.0", "yansi", ] -[[package]] -name = "rocket-apitoken" -version = "0.1.0" -source = "git+https://github.com/kvinwang/rocket-apitoken?branch=dev#7b5c1e864217b928e08913886c1a3ddd849e254c" -dependencies = [ - "rocket", -] - [[package]] name = "rocket-vsock-listener" version = "0.6.0" @@ -5699,7 +6427,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5830,9 +6558,9 @@ checksum = "6518fc26bced4d53678a22d6e423e9d8716377def84545fe328236e3af070e7f" [[package]] name = "s2n-codec" -version = "0.81.0" +version = "0.86.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d197a3c92bbe21fc00ba8366f6ba14edb8685316b6c8c14c622d3aba0a3816d8" +checksum = "2f2c33ef4ebeea0bd1fcdd31e3276d63e98ec4775b88d9695f9d6fe415a89842" dependencies = [ "byteorder", "bytes", @@ -5841,15 +6569,13 @@ dependencies = [ [[package]] name = "s2n-quic" -version = "1.81.0" +version = "1.86.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8728244102e791769cebe44a4abace966d8826f3266e9691c4233f47921b94b8" +checksum = "ae23e460d91121a2be1c7657715691a4643f36ae39702c219a2f41bb57c337c8" dependencies = [ "bytes", "cfg-if", - "cuckoofilter", "futures", - "hash_hasher", "rand 0.10.1", "s2n-codec", "s2n-quic-core", @@ -5865,9 +6591,9 @@ dependencies = [ [[package]] name = "s2n-quic-core" -version = "0.81.0" +version = "0.86.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cc69861a4909ea508b26309504899f4b0f77bb35348f6a36b7de9a28b1a4b92" +checksum = "8037d3498dcb9c6a473258328542953001e04d8f81e98163c7c708567dbdc3ad" dependencies = [ "atomic-waker", "byteorder", @@ -5887,9 +6613,9 @@ dependencies = [ [[package]] name = "s2n-quic-crypto" -version = "0.81.0" +version = "0.86.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3ce7f399a87be4b49d76895cdddb987620d34f334072d011bcac913d20fe69" +checksum = "e27dadbcbe234aa315f907f694688f9fb105b3fcb214feea24428b0faaea4b80" dependencies = [ "aws-lc-rs", "cfg-if", @@ -5913,24 +6639,24 @@ dependencies = [ [[package]] name = "s2n-quic-platform" -version = "0.81.0" +version = "0.86.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa9004809ae3a778b8e015581a47e9fb389f9ec230456a24b81c6287b000fefe" +checksum = "1bc6077b6164227a75475c3eacee00600b77fb5d3ae62e393113717e67e97e61" dependencies = [ "cfg-if", "futures", "lazy_static", "libc", "s2n-quic-core", - "socket2", + "socket2 0.6.4", "tokio", ] [[package]] name = "s2n-quic-rustls" -version = "0.81.0" +version = "0.86.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf7c34876c77f7560ee4385cd5ff0510acade2eb66dc237a45f7c63d2e7f1af3" +checksum = "ce1839226aef2a5ca6c3fa66eee2df02f715b9ed24a036328fcc39ecbb9014a4" dependencies = [ "bytes", "rustls", @@ -5942,9 +6668,9 @@ dependencies = [ [[package]] name = "s2n-quic-transport" -version = "0.81.0" +version = "0.86.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1ddd739c1776770dd2ab0b33da1cf372a395500252ae5250c08e2d6bf51b38f" +checksum = "c176d5f5c10a17455b1d4a7e7cf28bb965adbcd1bfbe3cbe5f5f3a65c102ed08" dependencies = [ "bytes", "futures-channel", @@ -5960,11 +6686,12 @@ dependencies = [ [[package]] name = "safe-write" -version = "0.1.2" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8066891189c8e6c7d189c4c19d841721606b4cac7212160ed3b3fa97d448fbab" +checksum = "b5a9dc0fc219eaa0265dbbee50170a469de86252d30b946c265c396b065b7f9f" dependencies = [ "fs-err", + "tempfile", ] [[package]] @@ -6168,6 +6895,7 @@ name = "serde-duration" version = "0.6.0" dependencies = [ "serde", + "serde_json", ] [[package]] @@ -6257,6 +6985,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_qs" version = "0.13.0" @@ -6316,9 +7055,35 @@ dependencies = [ "schemars 1.2.1", "serde_core", "serde_json", + "serde_with_macros", "time", ] +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.14.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sev" version = "6.0.0" @@ -6365,9 +7130,13 @@ dependencies = [ "anyhow", "hex", "moka", + "pem", "reqwest", + "rustls-pki-types", + "rustls-webpki", "sev", "tokio", + "x509-parser 0.16.0", ] [[package]] @@ -6521,6 +7290,12 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "simple-bytes" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c11532d9d241904f095185f35dcdaf930b1427a94d5b01d7002d74ba19b44cc4" + [[package]] name = "siphasher" version = "1.0.3" @@ -6570,6 +7345,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.4" @@ -6598,6 +7383,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "spki" version = "0.7.3" @@ -6688,8 +7479,8 @@ dependencies = [ "bon", "clap", "dashmap", + "dstack-build-info", "fs-err", - "git-version", "libc", "load_config", "nix 0.29.0", @@ -7060,7 +7851,7 @@ dependencies = [ "parking_lot 0.12.5", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.4", "tokio-macros", "windows-sys 0.61.2", ] @@ -7106,7 +7897,6 @@ dependencies = [ "bytes", "futures-core", "futures-sink", - "futures-util", "pin-project-lite", "tokio", ] @@ -7208,6 +7998,7 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -7280,7 +8071,7 @@ dependencies = [ "tokio", "tpm-types", "tracing", - "x509-parser", + "x509-parser 0.16.0", ] [[package]] @@ -7311,6 +8102,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -7414,6 +8206,12 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -7442,6 +8240,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.7.1" @@ -7498,10 +8302,21 @@ checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" dependencies = [ "getrandom 0.4.2", "js-sys", + "rand 0.10.1", "serde_core", "wasm-bindgen", ] +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "valuable" version = "0.1.1" @@ -7526,6 +8341,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "vsock" version = "0.5.4" @@ -7545,6 +8366,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -7564,12 +8394,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -7706,6 +8530,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "wavekv" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99f5f24dfab68f43806c0a15c3b87d9cde250dfc876cae3ec34642427becf336" +dependencies = [ + "anyhow", + "bincode 2.0.1", + "chrono", + "crc32fast", + "dashmap", + "fs-err", + "futures", + "hex", + "rmp-serde", + "serde", + "serde-human-bytes", + "serde_json", + "sha2 0.10.9", + "tokio", + "tracing", +] + [[package]] name = "web-sys" version = "0.3.99" @@ -7960,15 +8807,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.60.2" @@ -8300,6 +9138,18 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wrapcenum-derive" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76ff259533532054cfbaefb115c613203c73707017459206380f03b3b3f266e" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "writeable" version = "0.6.3" @@ -8345,18 +9195,36 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" dependencies = [ - "asn1-rs", + "asn1-rs 0.6.2", "data-encoding", - "der-parser", + "der-parser 9.0.0", "lazy_static", "nom", - "oid-registry", + "oid-registry 0.7.1", "ring", "rusticata-macros", "thiserror 1.0.69", "time", ] +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs 0.7.2", + "aws-lc-rs", + "data-encoding", + "der-parser 10.0.0", + "lazy_static", + "nom", + "oid-registry 0.8.1", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + [[package]] name = "xattr" version = "1.6.1" @@ -8419,6 +9287,16 @@ dependencies = [ "time", ] +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] + [[package]] name = "yoke" version = "0.8.2" diff --git a/dstack/Cargo.toml b/dstack/Cargo.toml index 1fd8daeeb..6223e773e 100644 --- a/dstack/Cargo.toml +++ b/dstack/Cargo.toml @@ -36,6 +36,7 @@ members = [ "tee-simulator", "vmm", "vmm/rpc", + "vmm/netd-rpc", "gateway", "gateway/rpc", "certbot", @@ -59,14 +60,25 @@ members = [ "serde-duration", "dstack-mr", "dstack-mr/cli", + "nvattest", + "nvidia-attest-proxy", "verifier", "size-parser", - "port-forward", "crates/dstack-cli-core", "crates/dstack-cli", + "crates/dstack-volume", "crates/dstackup", "crates/dstack-auth", + "crates/api-auth", + "crates/build-info", + "crates/mock-attestation", + "crates/qemu-acpi", ] +# Vendored third-party crates are path dependencies but deliberately not members: +# `--all-features` applies to members, and ktls declares its `ring` and +# `aws_lc_rs` features mutually exclusive, so membership makes +# `cargo test --all-features` fail to compile. +exclude = ["vendor/ktls"] resolver = "2" [workspace.dependencies] @@ -77,8 +89,12 @@ dstack-gateway-rpc = { path = "gateway/rpc" } dstack-kms-rpc = { path = "kms/rpc" } dstack-guest-agent-rpc = { path = "guest-agent/rpc" } dstack-vmm-rpc = { path = "vmm/rpc" } +dstack-netd-rpc = { path = "vmm/netd-rpc" } dstack-cli-core = { path = "crates/dstack-cli-core" } -dstack-port-forward = { path = "port-forward" } +dstack-volume = { path = "crates/dstack-volume" } +dstack-api-auth = { path = "crates/api-auth" } +dstack-build-info = { path = "crates/build-info" } +qemu-acpi = { path = "crates/qemu-acpi" } cc-eventlog = { path = "cc-eventlog" } supervisor = { path = "supervisor" } supervisor-client = { path = "supervisor/client" } @@ -108,7 +124,8 @@ serde-duration = { path = "serde-duration" } dstack-mr = { path = "dstack-mr" } dstack-verifier = { path = "verifier", default-features = false } size-parser = { path = "size-parser" } -wavekv = "1.0.0" +nvattest = { path = "nvattest" } +wavekv = "2.1" # Core dependencies anyhow = { version = "1.0.97", default-features = false } @@ -123,16 +140,21 @@ fs-err = "3.1.0" fuser = "0.16.0" path-absolutize = "3.1.1" futures = "0.3.31" -git-version = "0.3.9" libc = "0.2.171" log = "0.4.26" moka = { version = "0.12.15", default-features = false, features = ["sync"] } notify = "8.0.0" +nvml-wrapper = "0.12.1" rand = "0.8.5" +regorus = { version = "0.10.1", default-features = false, features = ["full-opa", "arc"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } -safe-write = "0.1.2" +safe-write = "0.2.0" +rustix = { version = "0.38", features = ["fs"] } nix = "0.29.0" +# Vendored: 6.0.2 does not build for musl. See vendor/README.md. +ktls = { path = "vendor/ktls" } +socket2 = { version = "0.5", features = ["all"] } sd-notify = "0.4.5" listenfd = "1.0" jemallocator = "0.5.4" @@ -142,6 +164,7 @@ flate2 = "1.1" borsh = { version = "1.5.7", default-features = false, features = ["derive"] } bon = { version = "3.4.0", default-features = false } base64 = "0.22.1" +bcrypt = "0.17.1" hex = { version = "0.4.3", default-features = false } hex_fmt = "0.3.0" hex-literal = "1.0.0" @@ -153,9 +176,13 @@ scale = { version = "3.7.4", package = "parity-scale-codec", features = [ ] } serde = { version = "1.0.228", features = ["derive"], default-features = false } serde-human-bytes = "0.1.2" -semver = "1.0.28" +serde_with = "3.14.0" serde_jcs = "0.2.0" rmp-serde = "1.3.1" +# The msgpack marker table, shared with rmp-serde rather than hand-copied. +rmp = "0.8.15" +# Test-only: a full msgpack decoder to differential-test the KV field walker. +rmpv = "1.3.1" serde_json = { version = "1.0.140", default-features = false } serde_ini = "0.2.0" toml = "0.8.20" @@ -206,7 +233,7 @@ url = "2.5" # Cryptography/Security aes-gcm = "0.10.3" curve25519-dalek = "4.1.3" -dcap-qvl = "0.5.2" +dcap-qvl = "0.5.3" dcap-qvl-webpki = "0.103.4" elliptic-curve = { version = "0.13.8", features = ["pkcs8"] } getrandom = "0.3.1" @@ -236,7 +263,7 @@ ez-hash = "1.1.0" # Certificate/DNS hickory-resolver = "0.26.1" -instant-acme = "0.7.2" +instant-acme = "0.8.5" pem = "3.0" rcgen = { version = "0.13.2", features = ["pem"] } x509-parser = "0.16.0" @@ -254,6 +281,7 @@ enum_dispatch = "0.3.13" insta = "1.42.2" num_enum = "0.7.3" thiserror = "2.0.12" +acpi_tables = "0.2.1" derive_more = "2.1.1" tempfile = "3.18.0" @@ -269,6 +297,7 @@ strip-ansi-escapes = "0.2.1" tailf = "0.1.2" time = "0.3.47" uuid = { version = "1.15.1", features = ["v4"] } +wait-timeout = "0.2" which = "7.0.2" smallvec = "1.14.0" cmd_lib = "1.9.5" diff --git a/dstack/build/shared/build-lib.sh b/dstack/build/shared/build-lib.sh index d24d368d4..51c7dcb50 100755 --- a/dstack/build/shared/build-lib.sh +++ b/dstack/build/shared/build-lib.sh @@ -6,12 +6,28 @@ # Shared build library for reproducible Docker image builds. # +# Release CI runs the very same component build-image.sh that a user runs by +# hand, so everything that ends up in the published image -- layers, config +# labels and manifest annotations -- is reproducible outside CI. +# # Expected variables (set by the sourcing script): # REPO_ROOT - absolute path to the monorepo root # CONTEXT_DIR - Docker build context directory # DOCKERFILE - path to the Dockerfile # GIT_REV - git revision to build # DSTACK_SRC_URL - git URL for dstack source +# +# Optional variables: +# IMAGE_VERSION - version recorded in the image metadata (default: dev). +# Release CI passes the tag version; pass the same value to +# reproduce a published image. +# IMAGE_SOURCE_URL - repository URL recorded in the image metadata +# PUSH - non-empty to publish after local validation +# OCI_TAR - path to also write an OCI archive to. The manifest digest +# of that archive is what a registry reports, so this is the +# way to check a local rebuild against a published digest. +# METADATA_FILE - path buildx writes its build metadata (incl. digest) to +# NO_CACHE - non-empty to build without the layer cache set -euo pipefail @@ -36,10 +52,65 @@ extract_packages() { >"$pkg_list_file" } +# Print the OCI metadata for a component image as "key=value" lines. +# +# This is the single source of truth for that metadata: docker_build feeds the +# result to the config labels, the manifest annotations and the in-image +# /etc//build-info file, so the three can never disagree. +# +# $1 - image title, e.g. dstack-kms +# $2 - one-line description +# $3 - component directory within the repo, e.g. dstack/kms +# $4 - documentation file within the repo, e.g. dstack/kms/README.md +image_metadata() { + local title=$1 + local description=$2 + local component_dir=$3 + local documentation=$4 + + local source_url=${IMAGE_SOURCE_URL:-${DSTACK_SRC_URL%.git}} + + # Read the pinned base image out of the Dockerfile's last FROM so the + # base.name/base.digest claims cannot drift from the layer they describe. + local base_ref + base_ref=$(awk '$1 == "FROM" { ref = $2 } END { print ref }' "$DOCKERFILE") + if [[ "$base_ref" != *@sha256:* ]]; then + echo "the final FROM in $DOCKERFILE must pin a digest; got '$base_ref'" >&2 + return 1 + fi + local base_name=${base_ref%@*} + local base_digest=${base_ref#*@} + # Normalize a bare official-image reference to its canonical form. + if [[ "$base_name" != */* ]]; then + base_name="docker.io/library/$base_name" + fi + + printf '%s\n' \ + "org.opencontainers.image.title=$title" \ + "org.opencontainers.image.description=$description" \ + "org.opencontainers.image.source=$source_url" \ + "org.opencontainers.image.revision=$GIT_REV" \ + "org.opencontainers.image.version=${IMAGE_VERSION:-dev}" \ + "org.opencontainers.image.url=$source_url/tree/$GIT_REV/$component_dir" \ + "org.opencontainers.image.documentation=$source_url/blob/$GIT_REV/$documentation" \ + "org.opencontainers.image.licenses=Apache-2.0" \ + "org.opencontainers.image.base.name=$base_name" \ + "org.opencontainers.image.base.digest=$base_digest" +} + +# docker_build <tags> [target] [pkg_list_file] [metadata] [mode] +# +# tags - newline-separated list of image tags +# target - build stage to stop at, empty for the final image +# pkg_list_file - where to record the installed Debian packages +# metadata - "key=value" lines from image_metadata, empty to record none +# mode - load (default), or export after package validation docker_build() { - local image_name=$1 + local tags=$1 local target=${2:-} local pkg_list_file=${3:-} + local metadata=${4:-} + local mode=${5:-load} local commit_timestamp commit_timestamp=$(git -C "$REPO_ROOT" show -s --format=%ct "$GIT_REV") @@ -47,26 +118,84 @@ docker_build() { local args=( --builder "$BUILDKIT_BUILDER" --progress=plain - --output "type=docker,name=$image_name,rewrite-timestamp=true" + # The Rust stages cross-compile to x86_64-unknown-linux-musl, so pin the + # platform rather than inheriting the host's -- an arm64 workstation must + # still reproduce the released amd64 image. + --platform linux/amd64 + # BuildKit attaches a provenance attestation by default when pushing, + # which carries build timestamps and turns the pushed tag into an index. + # Both would make the digest unreproducible. + --provenance=false --build-context "build-shared=$BUILD_SHARED_DIR" --build-arg "SOURCE_DATE_EPOCH=$commit_timestamp" --build-arg "DSTACK_REV=$GIT_REV" --build-arg "DSTACK_SRC_URL=$DSTACK_SRC_URL" ) - if [ -n "${NO_CACHE:-}" ]; then - args+=(--no-cache) + local tag + while IFS= read -r tag; do + [ -n "$tag" ] || continue + args+=(--tag "$tag") + done <<<"$tags" + + if [ -n "$metadata" ]; then + local pair + while IFS= read -r pair; do + [ -n "$pair" ] || continue + args+=(--label "$pair" --annotation "manifest:$pair") + done <<<"$metadata" + args+=(--build-arg "IMAGE_METADATA=$metadata") fi + # A validation build has only a Docker exporter. Never mix it with an OCI + # exporter: on classic Docker stores its digest overwrites the OCI digest in + # BuildKit's metadata response. Export the validated, cached result separately. + local outputs=() + case "$mode" in + load) + outputs+=("type=docker,rewrite-timestamp=true") + if [ -n "${NO_CACHE:-}" ]; then + args+=(--no-cache) + fi + ;; + export) + if [ -n "$target" ] || [ -n "$pkg_list_file" ]; then + echo "only validated final images may be exported" >&2 + return 1 + fi + if [ -n "${OCI_TAR:-}" ]; then + outputs+=("type=oci,oci-mediatypes=true,rewrite-timestamp=true,dest=$OCI_TAR") + fi + if [ -n "${PUSH:-}" ]; then + outputs+=("type=image,push=true,oci-mediatypes=true,rewrite-timestamp=true") + fi + if [ "${#outputs[@]}" -eq 0 ]; then + # METADATA_FILE alone still reports the registry-compatible OCI + # digest, not the manifest produced by a classic Docker exporter. + outputs+=("type=image,push=false,oci-mediatypes=true,rewrite-timestamp=true") + fi + if [ -n "${METADATA_FILE:-}" ]; then + args+=(--metadata-file "$METADATA_FILE") + fi + ;; + *) + echo "unknown build mode: $mode" >&2 + return 1 + ;; + esac + if [ -n "$target" ]; then args+=(--target "$target") fi - docker buildx build "${args[@]}" \ - --file "$DOCKERFILE" \ - "$CONTEXT_DIR" + local output + for output in "${outputs[@]}"; do + docker buildx build "${args[@]}" --output "$output" \ + --file "$DOCKERFILE" \ + "$CONTEXT_DIR" + done - extract_packages "$image_name" "$pkg_list_file" + extract_packages "$(head -n1 <<<"$tags")" "$pkg_list_file" } # Verify that pinned-packages files haven't changed (idempotency check). @@ -81,3 +210,30 @@ check_clean_tree() { exit 1 fi } + +# Build and validate all package lists before exporting or updating remote tags. +# The export reuses the validated cache even when NO_CACHE was requested for the +# validation builds. check_clean_tree ensures extraction did not change inputs. +build_component() { + local tags=$1 + local builder_tag=$2 + local builder_target=$3 + local shared_dir=$4 + local metadata=$5 + + if [ -n "${PUSH:-}${OCI_TAR:-}${METADATA_FILE:-}" ]; then + # Do not let package extraction erase pre-existing changes and make an + # unvalidated set of build inputs appear clean before publication. + check_clean_tree "$shared_dir" + fi + + # Build the intermediate stage first: with NO_CACHE it must not replace the + # final image's cached dependencies between validation and export. + docker_build "$builder_tag" "$builder_target" "$shared_dir/builder-pinned-packages.txt" + docker_build "$tags" "" "$shared_dir/pinned-packages.txt" "$metadata" + check_clean_tree "$shared_dir" + + if [ -n "${PUSH:-}${OCI_TAR:-}${METADATA_FILE:-}" ]; then + docker_build "$tags" "" "" "$metadata" export + fi +} diff --git a/dstack/build/shared/config-qemu.sh b/dstack/build/shared/config-qemu.sh deleted file mode 100755 index 94174a585..000000000 --- a/dstack/build/shared/config-qemu.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> -# -# SPDX-License-Identifier: Apache-2.0 - -BUILD_DIR="$1" -PREFIX="$2" -if [ -z "$BUILD_DIR" ]; then - echo "Usage: $0 <build-directory>" - exit 1 -fi - -mkdir -p "$BUILD_DIR" -cd "$BUILD_DIR" - -export SOURCE_DATE_EPOCH=$(git -C .. log -1 --pretty=%ct) -export CFLAGS="-DDUMP_ACPI_TABLES -Wno-builtin-macro-redefined -D__DATE__=\"\" -D__TIME__=\"\" -D__TIMESTAMP__=\"\"" -export LDFLAGS="-Wl,--build-id=none" - -../configure \ - --prefix="$PREFIX" \ - --target-list=x86_64-softmmu \ - --disable-werror - -echo "" -echo "Build configured for reproducibility in $BUILD_DIR" -echo "To build, run: cd $BUILD_DIR && make" diff --git a/dstack/build/shared/tests/test_build_lib.py b/dstack/build/shared/tests/test_build_lib.py new file mode 100644 index 000000000..4e907a619 --- /dev/null +++ b/dstack/build/shared/tests/test_build_lib.py @@ -0,0 +1,268 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise release orchestration without Docker or network access. + +Run with: python3 -m unittest discover -s dstack/build/shared/tests -v +""" + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +COMPONENTS = ("kms/dstack-app", "gateway/dstack-app", "verifier") +MOCK_DOCKER = r""" +import json +import os +import sys + +args = sys.argv[1:] +with open(os.environ["CALLS"], "a") as log: + log.write(json.dumps(args) + "\n") +if args[:2] == ["buildx", "build"]: + stage = "builder" if "--target" in args else "final" + if os.environ.get("FAIL_BUILD") == stage: + sys.exit(1) + if "--metadata-file" in args: + with open(args[args.index("--metadata-file") + 1], "w") as out: + json.dump({"containerimage.digest": "sha256:" + "1" * 64}, out) +elif args[0] == "run": + stage = "builder" if args[4].endswith("-builder-temp") else "final" + if os.environ.get("FAIL_EXTRACT") == stage: + sys.exit(1) + print("package=" + ("2" if os.environ.get("DRIFT") == stage else "1")) +""" + + +class BuildTests(unittest.TestCase): + """Check the real component entry points with a recording Docker executable.""" + + def setUp(self): + """Create a minimal tracked checkout and an annotated release tag.""" + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + shared = self.root / "dstack/build/shared" + shared.mkdir(parents=True) + shutil.copy(ROOT / "dstack/build/shared/build-lib.sh", shared) + for component in COMPONENTS: + builder = self.root / "dstack" / component / "builder" + (builder / "shared").mkdir(parents=True) + shutil.copy(ROOT / "dstack" / component / "builder/build-image.sh", builder) + (builder / "Dockerfile").write_text( + "FROM debian:bookworm@sha256:" + "0" * 64 + "\n" + ) + for name in ("pinned-packages.txt", "builder-pinned-packages.txt"): + (builder / "shared" / name).write_text("package=1\n") + self.git("init", "-q") + self.git("add", ".") + tree = self.git("write-tree") + # Synthetic objects avoid depending on or overriding any Git identity. + identity = "Build Fixture <fixture@example.invalid> 1700000000 +0000" + self.commit = self.git( + "hash-object", + "-t", + "commit", + "-w", + "--stdin", + input=f"tree {tree}\nauthor {identity}\ncommitter {identity}\n\nFixture\n", + ) + self.git("update-ref", "HEAD", self.commit) + tag = self.git( + "hash-object", + "-t", + "tag", + "-w", + "--stdin", + input=f"object {self.commit}\ntype commit\ntag release\ntagger {identity}\n\nRelease\n", + ) + self.git("update-ref", "refs/tags/release", tag) + bin_dir = self.root / "bin" + bin_dir.mkdir() + docker = bin_dir / "docker" + docker.write_text(f"#!{sys.executable}\n" + MOCK_DOCKER) + docker.chmod(0o755) + self.calls = self.root / "calls.jsonl" + self.metadata = self.root / "metadata.json" + self.env = { + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "CALLS": str(self.calls), + "GIT_REV": "release", + "IMAGE_VERSION": "1.0", + "PUSH": "", + "OCI_TAR": "", + "METADATA_FILE": "", + "NO_CACHE": "", + "FAIL_BUILD": "", + "FAIL_EXTRACT": "", + "DRIFT": "", + } + + def git(self, *args, input=None): + """Run Git only against the disposable fixture repository.""" + return subprocess.check_output( + ["git", "-C", str(self.root), *args], + input=input, + text=True, + ).strip() + + def run_build(self, component="kms/dstack-app", **env): + """Run a real entry point and return its result and recorded build calls.""" + self.calls.unlink(missing_ok=True) + result = subprocess.run( + [ + "bash", + str(self.root / "dstack" / component / "builder/build-image.sh"), + "example/probe:1.0", + "example/probe:latest", + ], + env={**self.env, **env}, + text=True, + capture_output=True, + ) + calls = [json.loads(line) for line in self.calls.read_text().splitlines()] + builds = [call for call in calls if call[:2] == ["buildx", "build"]] + return result, builds + + def test_local_build_peels_annotated_tags(self): + """All components resolve annotated tags and load only the two stages.""" + for component in COMPONENTS: + with self.subTest(component=component): + result, builds = self.run_build(component) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(builds), 2) + for build in builds: + self.assertIn(f"DSTACK_REV={self.commit}", build) + self.assertIn("SOURCE_DATE_EPOCH=1700000000", build) + self.assertEqual(build.count("--output"), 1) + self.assertIn("type=docker,rewrite-timestamp=true", build) + self.assertNotIn("--metadata-file", build) + + def test_exports_are_separate_and_follow_validation(self): + """A no-cache request does not force a fresh build after validation.""" + result, builds = self.run_build( + PUSH="1", + OCI_TAR=str(self.root / "image.tar"), + METADATA_FILE=str(self.metadata), + NO_CACHE="1", + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(builds), 4) + self.assertIn("--target", builds[0]) + self.assertNotIn("--target", builds[1]) + for build in builds[:2]: + self.assertIn("--no-cache", build) + self.assertNotIn("--metadata-file", build) + for build in builds[2:]: + self.assertEqual(build.count("--output"), 1) + self.assertNotIn("--no-cache", build) + self.assertNotIn("--target", build) + self.assertNotIn("type=docker,rewrite-timestamp=true", build) + self.assertIn("--metadata-file", build) + self.assertIn("example/probe:1.0", build) + self.assertIn("example/probe:latest", build) + self.assertTrue( + builds[2][builds[2].index("--output") + 1].startswith("type=oci,") + ) + self.assertIn( + "type=image,push=true,oci-mediatypes=true,rewrite-timestamp=true", builds[3] + ) + self.assertTrue(self.metadata.exists()) + + def test_failures_never_publish(self): + """Build errors, extraction errors and either drifting list block export.""" + for variable in ("FAIL_BUILD", "FAIL_EXTRACT", "DRIFT"): + for stage in ("final", "builder"): + with self.subTest(variable=variable, stage=stage): + self.git("restore", ".") + result, builds = self.run_build( + PUSH="1", + METADATA_FILE=str(self.metadata), + **{variable: stage}, + ) + self.assertNotEqual(result.returncode, 0) + for build in builds: + self.assertIn("type=docker,rewrite-timestamp=true", build) + self.assertNotIn("--metadata-file", build) + self.assertFalse(self.metadata.exists()) + + def test_dirty_inputs_block_publication_before_building(self): + """Extraction must not be allowed to erase dirty publication inputs.""" + pins = self.root / "dstack/kms/dstack-app/builder/shared/pinned-packages.txt" + pins.write_text("package=2\n") + result, builds = self.run_build(PUSH="1") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(builds, []) + self.assertEqual(pins.read_text(), "package=2\n") + + def test_metadata_only_uses_oci_image_exporter(self): + """Requesting metadata alone never reports a Docker schema manifest.""" + result, builds = self.run_build(METADATA_FILE=str(self.metadata)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(builds), 3) + self.assertIn( + "type=image,push=false,oci-mediatypes=true,rewrite-timestamp=true", + builds[-1], + ) + + +class WorkflowTests(unittest.TestCase): + """Exercise the shell version parser from each release workflow.""" + + def test_version_validation_and_release_refs(self): + """Accept release versions, reject shell syntax, and bind checkout/release tags.""" + for component in ("kms", "gateway", "verifier"): + workflow = (ROOT / f".github/workflows/{component}-release.yml").read_text() + parser = textwrap.dedent( + workflow.split(" run: |\n", 1)[1].split("\n\n", 1)[0] + ) + self.assertIn("tag_name: ${{ env.RELEASE_TAG }}", workflow) + self.assertIn("target_commitish: ${{ env.GIT_REV }}", workflow) + self.assertIn("&& env.RELEASE_TAG || github.sha", workflow) + for event in ("push", "workflow_dispatch"): + for version in ( + "1.0-rc1", + "", + "bad;echo injected", + "bad\nNAME=value", + "x" * 129, + ): + with self.subTest( + component=component, event=event, version=version + ): + with tempfile.NamedTemporaryFile() as output: + result = subprocess.run( + ["bash", "-euc", parser], + capture_output=True, + text=True, + env={ + **os.environ, + "GITHUB_EVENT_NAME": event, + "GITHUB_REF": f"refs/tags/{component}-v{version}" + if event == "push" + else "refs/heads/next", + "DISPATCH_VERSION": version, + "GITHUB_ENV": output.name, + }, + ) + self.assertEqual( + result.returncode == 0, version == "1.0-rc1" + ) + if result.returncode == 0: + self.assertIn( + f"RELEASE_TAG={component}-v{version}\n", + Path(output.name).read_text(), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/dstack/cc-eventlog/Cargo.toml b/dstack/cc-eventlog/Cargo.toml index 2863760f7..5fa010cf2 100644 --- a/dstack/cc-eventlog/Cargo.toml +++ b/dstack/cc-eventlog/Cargo.toml @@ -13,12 +13,15 @@ license.workspace = true [dependencies] anyhow.workspace = true digest = "0.10.7" +dstack-types.workspace = true ez-hash.workspace = true fs-err.workspace = true hex.workspace = true +or-panic.workspace = true scale.workspace = true serde.workspace = true serde-human-bytes.workspace = true +serde_jcs = "0.2.0" serde_json = { workspace = true, features = ["alloc"] } sha2.workspace = true diff --git a/dstack/cc-eventlog/src/lib.rs b/dstack/cc-eventlog/src/lib.rs index 93bbc77ff..26664405e 100644 --- a/dstack/cc-eventlog/src/lib.rs +++ b/dstack/cc-eventlog/src/lib.rs @@ -2,7 +2,10 @@ // // SPDX-License-Identifier: Apache-2.0 -pub use runtime_events::{replay_events, RuntimeEvent}; +pub use dstack_types::EventLogVersion; +pub use runtime_events::{ + canonical_event_json_v2, replay_events, RuntimeEvent, DSTACK_RUNTIME_EVENT_TYPE, +}; pub use tdx::TdxEvent; mod codecs; diff --git a/dstack/cc-eventlog/src/runtime_events.rs b/dstack/cc-eventlog/src/runtime_events.rs index fa948f36c..404d6450e 100644 --- a/dstack/cc-eventlog/src/runtime_events.rs +++ b/dstack/cc-eventlog/src/runtime_events.rs @@ -3,7 +3,9 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::{Context, Result}; +use dstack_types::EventLogVersion; use fs_err as fs; +use or_panic::ResultOrPanic; use scale::{Decode, Encode}; use serde::{Deserialize, Serialize}; use serde_human_bytes::base64; @@ -14,6 +16,10 @@ use ez_hash::{Hasher, Sha256, Sha384}; /// The event type for dstack runtime events. /// This code is not defined in the TCG specification. /// See https://trustedcomputinggroup.org/wp-content/uploads/PC-ClientSpecific_Platform_Profile_for_TPM_2p0_Systems_v51.pdf +/// +/// V1 and V2 use the same event type; the digest format is distinguished by +/// `EventLogVersion` (carried on `RuntimeEvent`/`TdxEvent` or inferred from +/// the v2 canonical JSON content). pub const DSTACK_RUNTIME_EVENT_TYPE: u32 = 0x08000001; /// The path to the userspace TDX event log file. pub const RUNTIME_EVENT_LOG_FILE: &str = "/run/log/dstack/runtime_events.log"; @@ -26,11 +32,19 @@ pub struct RuntimeEvent { /// Event payload #[serde(with = "base64")] pub payload: Vec<u8>, + /// Event log version + #[serde(default, skip_serializing_if = "EventLogVersion::is_v1")] + #[codec(skip)] + pub version: EventLogVersion, } impl RuntimeEvent { - pub fn new(event: String, payload: Vec<u8>) -> Self { - Self { event, payload } + pub fn new(event: String, payload: Vec<u8>, version: EventLogVersion) -> Self { + Self { + event, + payload, + version, + } } pub fn read_all() -> Result<Vec<RuntimeEvent>> { @@ -97,21 +111,56 @@ impl RuntimeEvent { } /// Compute the digest of the event. + /// + /// - V1: `SHA(event_type_le || ":" || event_name || ":" || payload)` + /// - V2: `SHA(canonical_json({"name":"...","type":134217729,"payload":"hex..."}))` pub fn digest<H: Hasher>(&self) -> H::Output { - H::hash([ - &DSTACK_RUNTIME_EVENT_TYPE.to_ne_bytes()[..], - b":", - self.event.as_bytes(), - b":", - &self.payload, - ]) + H::hash([self.preimage().as_slice()]) + } + + /// The exact byte sequence that gets hashed to produce the digest. + /// + /// Useful for relying parties that want to verify the digest computation + /// or inspect event content without knowing the dstack schema. + /// + /// - V1: binary concatenation `event_type_le || ":" || name || ":" || payload` + /// - V2: UTF-8 bytes of the JCS canonical JSON + pub fn preimage(&self) -> Vec<u8> { + match self.version { + EventLogVersion::V1 => { + let mut buf = Vec::with_capacity(4 + 1 + self.event.len() + 1 + self.payload.len()); + buf.extend_from_slice(&DSTACK_RUNTIME_EVENT_TYPE.to_le_bytes()); + buf.push(b':'); + buf.extend_from_slice(self.event.as_bytes()); + buf.push(b':'); + buf.extend_from_slice(&self.payload); + buf + } + EventLogVersion::V2 => canonical_event_json_v2(&self.event, &self.payload).into_bytes(), + } } + /// The event type used when extending RTMR. Always `DSTACK_RUNTIME_EVENT_TYPE`. + /// Version is distinguished via `EventLogVersion`, not the event type. pub fn cc_event_type(&self) -> u32 { DSTACK_RUNTIME_EVENT_TYPE } } +/// Construct the JCS (RFC 8785) canonical JSON used as the v2 digest input. +/// +/// Keys and number/string formatting are handled by `serde_jcs` per RFC 8785. +/// Version is carried out-of-band via `RuntimeEvent::version`, not in the +/// hashed content. +pub fn canonical_event_json_v2(event: &str, payload: &[u8]) -> String { + let obj = serde_json::json!({ + "name": event, + "type": DSTACK_RUNTIME_EVENT_TYPE, + "payload": hex::encode(payload), + }); + serde_jcs::to_string(&obj).or_panic("canonical JSON serialization failed") +} + /// Replay event logs pub fn replay_events<H: Hasher>(eventlog: &[RuntimeEvent], to_event: Option<&str>) -> H::Output { let mut mr = H::zeros(); @@ -125,3 +174,221 @@ pub fn replay_events<H: Hasher>(eventlog: &[RuntimeEvent], to_event: Option<&str } mr } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn v1_digest_unchanged() { + let event = RuntimeEvent::new( + "app-id".to_string(), + vec![0xde, 0xad, 0xbe, 0xef], + EventLogVersion::V1, + ); + let digest = event.digest::<Sha384>(); + let expected = Sha384::hash([ + &DSTACK_RUNTIME_EVENT_TYPE.to_le_bytes()[..], + b":", + b"app-id", + b":", + &[0xde, 0xad, 0xbe, 0xef], + ]); + assert_eq!(digest, expected, "v1 digest must be backward compatible"); + } + + #[test] + fn v2_digest_is_canonical_json_hash() { + let event = RuntimeEvent::new( + "compose-hash".to_string(), + vec![0xab, 0xcd], + EventLogVersion::V2, + ); + let canonical = canonical_event_json_v2(&event.event, &event.payload); + assert_eq!( + canonical, + r#"{"name":"compose-hash","payload":"abcd","type":134217729}"# + ); + let digest = event.digest::<Sha384>(); + let expected = Sha384::hash([canonical.as_bytes()]); + assert_eq!(digest, expected); + } + + #[test] + fn v2_digest_differs_from_v1() { + let v1 = RuntimeEvent::new("test".to_string(), vec![1, 2, 3], EventLogVersion::V1); + let v2 = RuntimeEvent::new("test".to_string(), vec![1, 2, 3], EventLogVersion::V2); + assert_ne!( + v1.digest::<Sha384>(), + v2.digest::<Sha384>(), + "v1 and v2 digests must differ" + ); + } + + #[test] + fn v1_event_type() { + let event = RuntimeEvent::new("test".to_string(), vec![], EventLogVersion::V1); + assert_eq!(event.cc_event_type(), DSTACK_RUNTIME_EVENT_TYPE); + } + + #[test] + fn v2_event_type() { + // v2 uses the same event_type as v1 — version is carried separately + let event = RuntimeEvent::new("test".to_string(), vec![], EventLogVersion::V2); + assert_eq!(event.cc_event_type(), DSTACK_RUNTIME_EVENT_TYPE); + } + + #[test] + fn deserialize_v1_without_version_field() { + let json = r#"{"event":"app-id","payload":"AQID"}"#; + let event: RuntimeEvent = serde_json::from_str(json).unwrap(); + assert_eq!(event.version, EventLogVersion::V1); + assert_eq!(event.cc_event_type(), DSTACK_RUNTIME_EVENT_TYPE); + } + + #[test] + fn serde_roundtrip_preserves_version() { + let v2 = RuntimeEvent::new("test".to_string(), vec![1], EventLogVersion::V2); + let json = serde_json::to_string(&v2).unwrap(); + assert!(json.contains(r#""version":2"#), "v2 must serialize version"); + let decoded: RuntimeEvent = serde_json::from_str(&json).unwrap(); + assert_eq!(decoded.version, EventLogVersion::V2); + } + + #[test] + fn deserialize_without_version_defaults_to_v1() { + let json = r#"{"event":"test","payload":"AQ=="}"#; + let decoded: RuntimeEvent = serde_json::from_str(json).unwrap(); + assert_eq!(decoded.version, EventLogVersion::V1); + } + + #[test] + fn canonical_json_escapes_special_chars() { + let canonical = canonical_event_json_v2("event\"with\\special\nchars", &[0xff]); + // Exact bytewise output — JCS must be deterministic + assert_eq!( + canonical, + r#"{"name":"event\"with\\special\nchars","payload":"ff","type":134217729}"# + ); + } + + #[test] + fn canonical_json_keys_are_sorted_alphabetically() { + // JCS RFC 8785 requires keys sorted by UTF-16 code unit order. + // For ASCII keys, this is alphabetical. + let canonical = canonical_event_json_v2("test", &[0x01]); + let name_pos = canonical.find(r#""name":"#).unwrap(); + let payload_pos = canonical.find(r#""payload":"#).unwrap(); + let type_pos = canonical.find(r#""type":"#).unwrap(); + assert!(name_pos < payload_pos); + assert!(payload_pos < type_pos); + } + + #[test] + fn canonical_json_empty_event_and_payload() { + let canonical = canonical_event_json_v2("", &[]); + assert_eq!(canonical, r#"{"name":"","payload":"","type":134217729}"#); + } + + #[test] + fn canonical_json_idempotent() { + // Same input must always produce bytewise-identical output. + // HashMap randomization internally shouldn't affect output. + let reference = canonical_event_json_v2("compose-hash", &[0xde, 0xad, 0xbe, 0xef]); + for _ in 0..100 { + assert_eq!( + canonical_event_json_v2("compose-hash", &[0xde, 0xad, 0xbe, 0xef]), + reference + ); + } + } + + #[test] + fn canonical_json_non_ascii_unicode() { + // JCS requires UTF-8 output; non-ASCII characters that don't need + // escaping (i.e., not control chars, not " or \) must be emitted as-is. + let canonical = canonical_event_json_v2("测试-emoji-🦀", &[]); + // Event name should appear verbatim in the JSON (no \uXXXX escaping) + assert!(canonical.contains("测试-emoji-🦀"), "got: {canonical}"); + // Must still be parseable and roundtrip + let parsed: serde_json::Value = serde_json::from_str(&canonical).unwrap(); + assert_eq!(parsed["name"].as_str().unwrap(), "测试-emoji-🦀"); + } + + #[test] + fn canonical_json_control_character_escaping() { + // JCS (via RFC 8259) uses short escapes for \b \f \n \r \t and \uXXXX for other controls. + let canonical = canonical_event_json_v2("\x08\x0c\n\r\t\x01", &[]); + assert!( + canonical.contains(r#""name":"\b\f\n\r\t\u0001""#), + "got: {canonical}" + ); + } + + #[test] + fn canonical_json_payload_lowercase_hex() { + // Payload must be hex-encoded lowercase for determinism. + let canonical = canonical_event_json_v2("test", &[0xAB, 0xCD, 0xEF]); + assert!( + canonical.contains(r#""payload":"abcdef""#), + "got: {canonical}" + ); + } + + #[test] + fn canonical_json_is_valid_rfc8785_structure() { + // No whitespace, no trailing commas, proper JSON + let canonical = canonical_event_json_v2("x", &[0xff]); + assert!(!canonical.contains(' ')); + assert!(!canonical.contains('\n')); + assert!(!canonical.contains('\t')); + assert!(canonical.starts_with('{')); + assert!(canonical.ends_with('}')); + // Must parse back + let _: serde_json::Value = serde_json::from_str(&canonical).unwrap(); + } + + #[test] + fn mixed_v1_v2_replay() { + let events = vec![ + RuntimeEvent::new("app-id".to_string(), vec![1, 2], EventLogVersion::V1), + RuntimeEvent::new("compose-hash".to_string(), vec![3, 4], EventLogVersion::V2), + RuntimeEvent::new("instance-id".to_string(), vec![5, 6], EventLogVersion::V1), + ]; + let mr = replay_events::<Sha384>(&events, None); + // Replay manually to verify + let mut expected = Sha384::zeros(); + expected = Sha384::hash((expected, events[0].digest::<Sha384>())); + expected = Sha384::hash((expected, events[1].digest::<Sha384>())); + expected = Sha384::hash((expected, events[2].digest::<Sha384>())); + assert_eq!(mr, expected, "mixed v1/v2 replay must work correctly"); + } + + #[test] + fn scale_roundtrip_preserves_event_data() { + use scale::{Decode, Encode}; + // V1 event + let v1 = RuntimeEvent::new("test".to_string(), vec![1, 2, 3], EventLogVersion::V1); + let encoded = v1.encode(); + let decoded = RuntimeEvent::decode(&mut &encoded[..]).unwrap(); + assert_eq!(decoded.event, v1.event); + assert_eq!(decoded.payload, v1.payload); + // version is #[codec(skip)] so it defaults to V1 on decode + assert_eq!(decoded.version, EventLogVersion::V1); + } + + #[test] + fn scale_decode_old_format_without_version() { + use scale::{Decode, Encode}; + // Encode a current RuntimeEvent (version is skipped by codec), + // then decode — simulates reading data from before version was added + let original = + RuntimeEvent::new("app-id".to_string(), vec![0xaa, 0xbb], EventLogVersion::V2); + let encoded = original.encode(); + let decoded = RuntimeEvent::decode(&mut &encoded[..]).unwrap(); + assert_eq!(decoded.event, "app-id"); + assert_eq!(decoded.payload, vec![0xaa, 0xbb]); + // version is #[codec(skip)] so always decodes as default (V1) + assert_eq!(decoded.version, EventLogVersion::V1); + } +} diff --git a/dstack/cc-eventlog/src/tcg.rs b/dstack/cc-eventlog/src/tcg.rs index c6280abd6..5e8e6384c 100644 --- a/dstack/cc-eventlog/src/tcg.rs +++ b/dstack/cc-eventlog/src/tcg.rs @@ -395,6 +395,8 @@ impl TryFrom<TcgEvent> for TdxEvent { digest, event: Default::default(), event_payload: value.event.into(), + version: Default::default(), + preimage: None, }) } } diff --git a/dstack/cc-eventlog/src/tdx.rs b/dstack/cc-eventlog/src/tdx.rs index 615c32ee9..d4bd94e49 100644 --- a/dstack/cc-eventlog/src/tdx.rs +++ b/dstack/cc-eventlog/src/tdx.rs @@ -2,9 +2,11 @@ // // SPDX-License-Identifier: Apache-2.0 -use anyhow::Result; +use anyhow::{bail, Context, Result}; +use dstack_types::EventLogVersion; use scale::{Decode, Encode}; use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha384 as Sha384Digest}; use crate::{ runtime_events::{RuntimeEvent, DSTACK_RUNTIME_EVENT_TYPE}, @@ -27,7 +29,9 @@ pub const TDX_ACPI_DATA_EVENT_NAMES: [&str; 3] = [ /// and the raw event data. The IMR index is zero-based, unlike the TCG event log format /// which is one-based. /// -/// As for RTMR3, the digest extended is calculated as `sha384(event_type.to_ne_bytes() || b":" || event || b":" || event_payload)`. +/// For dstack runtime events (`event_type == DSTACK_RUNTIME_EVENT_TYPE`), the digest is: +/// - V1: `sha384(event_type_le || ":" || event || ":" || payload)` +/// - V2: `sha384(canonical_json({"name":"...","type":134217729,"payload":"hex..."}))` #[derive(Clone, Debug, Serialize, Deserialize, Encode, Decode)] pub struct TdxEvent { /// IMR index, starts from 0 @@ -42,6 +46,23 @@ pub struct TdxEvent { /// Event payload #[serde(with = "serde_human_bytes")] pub event_payload: Vec<u8>, + /// Event log version (for dstack runtime events). + /// Skipped by scale codec for binary compat with legacy attestations + /// (which only ever contain V1 events). + /// Serde skips serialization when V1 so existing JSON outputs stay clean. + #[serde(default, skip_serializing_if = "EventLogVersion::is_v1")] + #[codec(skip)] + pub version: EventLogVersion, + + /// Optional digest pre-image, hex-encoded. + /// + /// The exact bytes hashed to produce `digest`. V2 events exposed through + /// quote and attestation APIs always include it, allowing relying parties + /// to verify `sha384(hex_decode(preimage)) == digest`. + /// Never included in scale encoding (derivable from other fields). + #[serde(default, skip_serializing_if = "Option::is_none")] + #[codec(skip)] + pub preimage: Option<String>, } impl TdxEvent { @@ -52,6 +73,8 @@ impl TdxEvent { digest: vec![], event, event_payload, + version: EventLogVersion::default(), + preimage: None, } } @@ -62,9 +85,11 @@ impl TdxEvent { Self { imr: self.imr, event_type: self.event_type, - digest: Vec::new(), + digest: self.digest.clone(), event: self.event.clone(), event_payload: self.event_payload.clone(), + version: self.version, + preimage: self.preimage.clone(), } } else { Self { @@ -73,10 +98,26 @@ impl TdxEvent { digest: self.digest.clone(), event: self.event.clone(), event_payload: Vec::new(), + version: self.version, + preimage: self.preimage.clone(), } } } + /// Populate `preimage` with the digest pre-image. + /// + /// For runtime events, this is the byte sequence defined by V1/V2 digest algorithms. + /// For boot-time TCG events, the pre-image is inherent in the original log format + /// and not reconstructable from this struct, so `preimage` stays `None`. + pub fn fill_preimage(&mut self) { + if self.preimage.is_some() { + return; + } + if let Some(runtime_event) = self.to_runtime_event() { + self.preimage = Some(hex::encode(runtime_event.preimage())); + } + } + pub fn digest(&self) -> Vec<u8> { if let Some(runtime_event) = self.to_runtime_event() { return runtime_event.sha384_digest().to_vec(); @@ -89,23 +130,72 @@ impl TdxEvent { } pub fn to_runtime_event(&self) -> Option<RuntimeEvent> { - self.is_runtime_event().then_some(RuntimeEvent { + if !self.is_runtime_event() { + return None; + } + Some(RuntimeEvent { event: self.event.clone(), payload: self.event_payload.clone(), + version: self.version, }) } } impl From<RuntimeEvent> for TdxEvent { fn from(value: RuntimeEvent) -> Self { + let event_type = value.cc_event_type(); + let version = value.version; + let digest = value.sha384_digest().to_vec(); TdxEvent { imr: 3, - event_type: DSTACK_RUNTIME_EVENT_TYPE, - digest: value.sha384_digest().to_vec(), + event_type, + digest, event: value.event, event_payload: value.payload, + version, + preimage: None, + } + } +} + +/// Populate digest preimages for all V2 runtime events. +pub fn fill_v2_preimages(events: &mut [TdxEvent]) { + for event in events { + if matches!(event.version, EventLogVersion::V2) { + event.fill_preimage(); + } + } +} + +/// Validate the externally supplied digest preimage of every V2 runtime event. +/// +/// The preimage must be present, valid hex, hash to the advertised digest, and +/// equal the canonical representation reconstructed from the public event +/// fields. This binds RTMR replay and displayed fields to the same bytes. +pub fn validate_v2_preimages(events: &[TdxEvent]) -> Result<()> { + for (index, event) in events.iter().enumerate() { + if !event.is_runtime_event() || !matches!(event.version, EventLogVersion::V2) { + continue; + } + let supplied_hex = event + .preimage + .as_deref() + .with_context(|| format!("V2 runtime event {index} is missing its digest preimage"))?; + let supplied = hex::decode(supplied_hex) + .with_context(|| format!("V2 runtime event {index} has a malformed digest preimage"))?; + let advertised = Sha384Digest::digest(&supplied); + if advertised.as_slice() != event.digest.as_slice() { + bail!("V2 runtime event {index} digest does not match its preimage"); + } + let Some(runtime_event) = event.to_runtime_event() else { + continue; + }; + let canonical = runtime_event.preimage(); + if supplied != canonical { + bail!("V2 runtime event {index} preimage is not the canonical event representation"); } } + Ok(()) } pub fn is_tdx_acpi_data_event(event: &TdxEvent) -> bool { @@ -148,6 +238,8 @@ pub fn read_event_log() -> Result<Vec<TdxEvent>> { #[cfg(test)] mod tests { use super::*; + use ez_hash::{Hasher, Sha384}; + use sha2::Sha384 as Sha384Hasher; fn acpi_data_event(digest_byte: u8) -> TdxEvent { TdxEvent { @@ -156,6 +248,8 @@ mod tests { digest: vec![digest_byte; 48], event: String::new(), event_payload: TDX_ACPI_DATA_EVENT_PAYLOAD.to_vec(), + version: EventLogVersion::V1, + preimage: None, } } @@ -188,4 +282,124 @@ mod tests { assert!(events.iter().all(|event| event.imr <= 3)); assert!(events.iter().all(|event| event.digest().len() == 48)); } + + #[test] + fn fill_preimage_v1() { + let runtime = RuntimeEvent::new( + "compose-hash".to_string(), + vec![0xde, 0xad], + EventLogVersion::V1, + ); + let mut tdx: TdxEvent = runtime.into(); + assert_eq!(tdx.preimage, None); + tdx.fill_preimage(); + let input_hex = tdx.preimage.as_ref().expect("preimage populated"); + let input = hex::decode(input_hex).unwrap(); + // Hashing the preimage must reproduce the event digest + let actual = Sha384Hasher::digest(&input); + assert_eq!(actual.as_slice(), &tdx.digest); + } + + #[test] + fn fill_preimage_v2_is_canonical_json() { + let runtime = RuntimeEvent::new( + "compose-hash".to_string(), + vec![0xab, 0xcd], + EventLogVersion::V2, + ); + let mut tdx: TdxEvent = runtime.into(); + tdx.fill_preimage(); + let input_hex = tdx.preimage.as_ref().expect("preimage populated"); + let input = hex::decode(input_hex).unwrap(); + let input_str = std::str::from_utf8(&input).unwrap(); + // V2 preimage is the canonical JSON (version is carried out-of-band) + assert!(input_str.contains(r#""name":"compose-hash""#)); + assert!(input_str.contains(r#""type":134217729"#)); + assert!(input_str.contains(r#""payload":"abcd""#)); + assert!(!input_str.contains(r#""version""#)); + // And hashing it reproduces the digest + let actual = Sha384::hash([input.as_slice()]); + assert_eq!(actual.as_slice(), &tdx.digest); + } + + #[test] + fn stripped_v2_runtime_event_preserves_digest_binding() { + let mut event = v2_event(); + event.fill_preimage(); + + let stripped = event.stripped(); + + assert_eq!(stripped.digest, event.digest); + validate_v2_preimages(&[stripped]).expect("stripped V2 event remains verifiable"); + } + + #[test] + fn fill_preimage_skips_non_runtime_events() { + let mut boot_event = TdxEvent::new(0, 0x1, "EV_POST_CODE".to_string(), vec![1, 2, 3]); + boot_event.fill_preimage(); + assert_eq!(boot_event.preimage, None); + } + + #[test] + fn preimage_not_serialized_by_scale() { + use scale::{Decode, Encode}; + let runtime = RuntimeEvent::new("test".to_string(), vec![1, 2], EventLogVersion::V2); + let mut tdx: TdxEvent = runtime.into(); + tdx.fill_preimage(); + assert!(tdx.preimage.is_some()); + let encoded = tdx.encode(); + let decoded = TdxEvent::decode(&mut &encoded[..]).unwrap(); + // preimage is codec(skip) so it's None after round-trip + assert_eq!(decoded.preimage, None); + } + + #[test] + fn preimage_skipped_from_json_when_none() { + let runtime = RuntimeEvent::new("test".to_string(), vec![1], EventLogVersion::V1); + let tdx: TdxEvent = runtime.into(); + let json = serde_json::to_string(&tdx).unwrap(); + assert!(!json.contains("preimage")); + } + + fn v2_event() -> TdxEvent { + let mut event = TdxEvent::from(RuntimeEvent::new( + "app-id".into(), + b"fixture".to_vec(), + EventLogVersion::V2, + )); + event.fill_preimage(); + event + } + + #[test] + fn validates_v2_digest_preimage_before_use() { + validate_v2_preimages(&[v2_event()]).expect("valid V2 preimage"); + } + + #[test] + fn rejects_missing_or_malformed_v2_preimage() { + let mut missing = v2_event(); + missing.preimage = None; + assert!(validate_v2_preimages(&[missing]).is_err()); + + let mut malformed = v2_event(); + malformed.preimage = Some("not-hex".into()); + assert!(validate_v2_preimages(&[malformed]).is_err()); + } + + #[test] + fn rejects_v2_preimage_digest_mismatch() { + let mut event = v2_event(); + event.digest[0] ^= 1; + assert!(validate_v2_preimages(&[event]).is_err()); + } + + #[test] + fn rejects_noncanonical_v2_preimage_with_matching_digest() { + let mut event = v2_event(); + let supplied = br#"{"type":134217729,"name":"app-id","payload":"66697874757265"}"#; + event.preimage = Some(hex::encode(supplied)); + event.digest = Sha384Hasher::digest(supplied).to_vec(); + assert!(validate_v2_preimages(&[event]).is_err()); + } } diff --git a/dstack/cert-client/Cargo.toml b/dstack/cert-client/Cargo.toml index 089b927bf..e18f43ed4 100644 --- a/dstack/cert-client/Cargo.toml +++ b/dstack/cert-client/Cargo.toml @@ -17,3 +17,11 @@ ra-rpc = { workspace = true, features = ["client"] } ra-tls = { workspace = true, features = ["quote"] } serde_json.workspace = true tdx-attest.workspace = true +dstack-guest-agent-rpc.workspace = true +http-client = { workspace = true, features = ["prpc"] } +tokio.workspace = true + +[dev-dependencies] +hex.workspace = true +x509-parser.workspace = true +tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/dstack/cert-client/src/bin/dstack-kms-sign-cert-fixture.rs b/dstack/cert-client/src/bin/dstack-kms-sign-cert-fixture.rs new file mode 100644 index 000000000..734744723 --- /dev/null +++ b/dstack/cert-client/src/bin/dstack-kms-sign-cert-fixture.rs @@ -0,0 +1,94 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// SPDX-License-Identifier: Apache-2.0 +//! Generate a v2 KMS CSR whose key is bound to fresh guest attestation. + +use anyhow::{Context, Result}; +use dstack_guest_agent_rpc::v0::{dstack_guest_client::DstackGuestClient, RawQuoteArgs}; +use http_client::prpc::PrpcClient; +use ra_tls::{ + attestation::{PlatformEvidence, QuoteContentType, VersionedAttestation}, + cert::{CertConfig, CertConfigV2, CertSigningRequestV1, CertSigningRequestV2, Csr}, + rcgen::{KeyPair, PKCS_ECDSA_P256_SHA256}, +}; +use serde_json::json; + +fn hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + output.push(DIGITS[(byte >> 4) as usize] as char); + output.push(DIGITS[(byte & 0x0f) as usize] as char); + } + output +} + +#[tokio::main] +async fn main() -> Result<()> { + let key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256) + .context("failed to generate the case-scoped certificate key")?; + let pubkey = key.public_key_der(); + let report_data = QuoteContentType::RaTlsCert.to_report_data(&pubkey); + let address = dstack_types::dstack_agent_address(); + let client = DstackGuestClient::new(PrpcClient::new(address)); + let response = client + .attest(RawQuoteArgs { + report_data: report_data.to_vec(), + }) + .await + .context("failed to obtain key-bound guest attestation")?; + let attestation = VersionedAttestation::from_bytes(&response.attestation) + .context("failed to decode key-bound guest attestation")?; + let csr = CertSigningRequestV2 { + confirm: "please sign cert:".to_string(), + pubkey: pubkey.clone(), + config: CertConfigV2 { + org_name: Some("Dstack Test".to_string()), + subject: "kms-sign-cert.test".to_string(), + subject_alt_names: vec!["kms-sign-cert.test".to_string()], + usage_server_auth: true, + usage_client_auth: true, + ext_quote: true, + ext_app_info: true, + not_before: None, + not_after: None, + }, + attestation: attestation.clone(), + }; + let signature = csr.signed_by(&key).context("failed to sign the v2 CSR")?; + let (quote, event_log) = match attestation.clone().into_v1().platform { + PlatformEvidence::Tdx { quote, event_log } => (quote, event_log), + _ => anyhow::bail!("legacy v1 CSR fixture requires TDX evidence"), + }; + let csr_v1 = CertSigningRequestV1 { + confirm: "please sign cert:".to_string(), + pubkey: pubkey.clone(), + config: CertConfig { + org_name: Some("Dstack Test".to_string()), + subject: "kms-sign-cert.test".to_string(), + subject_alt_names: vec!["kms-sign-cert.test".to_string()], + usage_server_auth: true, + usage_client_auth: true, + ext_quote: true, + }, + quote, + event_log: serde_json::to_vec(&event_log) + .context("failed to encode the legacy TDX event log")?, + }; + let signature_v1 = csr_v1 + .signed_by(&key) + .context("failed to sign the v1 CSR")?; + println!( + "{}", + json!({ + "api_version": 2, + "csr": hex(&csr.to_vec()), + "signature": hex(&signature), + "csr_v1": hex(&csr_v1.data_to_sign()), + "signature_v1": hex(&signature_v1), + "public_key": hex(&pubkey), + "subject": "kms-sign-cert.test", + "alt_name": "kms-sign-cert.test" + }) + ); + Ok(()) +} diff --git a/dstack/cert-client/src/lib.rs b/dstack/cert-client/src/lib.rs index 8b5b329a1..93a44909b 100644 --- a/dstack/cert-client/src/lib.rs +++ b/dstack/cert-client/src/lib.rs @@ -2,11 +2,16 @@ // // SPDX-License-Identifier: Apache-2.0 +use std::sync::Arc; + use anyhow::{Context, Result}; use dstack_kms_rpc::{kms_client::KmsClient, SignCertRequest}; use dstack_types::{AppKeys, KeyProvider}; use ra_rpc::client::{RaClient, RaClientConfig}; -use ra_tls::cert::{generate_ra_cert, CaCert, CertSigningRequestV2}; +use ra_tls::{ + attestation::{AttestationVerifier, VersionedAttestation}, + cert::{generate_ra_cert, CaCert, CertSigningRequestV2}, +}; pub enum CertRequestClient { Local { @@ -18,6 +23,38 @@ pub enum CertRequestClient { }, } +/// The app id a locally issued certificate should carry, if any. +/// +/// KMS stamps the app_id it verified into the certificate it returns. The local +/// CA branch has the same value available -- the CSR carries the attestation it +/// was derived from -- but used to drop it, so every certificate issued through +/// a local CA came back without the extension. Consumers that read the +/// extension and have no fallback therefore rejected all of them: +/// `AppIdValidator` in dstack-gateway's cluster sync client and the gateway's +/// own `ensure_from_gateway` both do exactly that, which left clustering +/// working under a KMS key provider and broken under every other one. (The +/// gateway's inbound sync routes happen to fall back to the app-info extension, +/// which is why the failure showed up on one side of the connection only.) +/// +/// Reading the attestation without verifying it is sound *here and only here*: +/// the sole caller is the guest agent signing a CSR it built itself, from its +/// own `certificate_attestation`, against a CA whose key it already holds. It +/// asserts nothing it could not assert anyway. A remote CSR must go through +/// KMS, which verifies the quote first and stamps `boot_info.app_id`. +/// +/// Best effort on purpose: an app whose attestation carries no app-id event +/// decodes to an empty one, and stamping that would make every such peer match +/// every other. Absent stays absent, and the peer rejects it as before. +fn local_ca_app_id(attestation: &VersionedAttestation) -> Option<Vec<u8>> { + attestation + .clone() + .into_v1() + .decode_app_info(false) + .ok() + .map(|info| info.app_id) + .filter(|app_id| !app_id.is_empty()) +} + impl CertRequestClient { pub async fn sign_csr( &self, @@ -26,8 +63,9 @@ impl CertRequestClient { ) -> Result<Vec<String>> { match self { CertRequestClient::Local { ca } => { + let app_id = local_ca_app_id(&csr.attestation); let cert = ca - .sign_csr(csr, None, "app:custom") + .sign_csr(csr, app_id.as_deref(), "app:custom") .context("Failed to sign certificate")?; Ok(vec![cert.pem(), ca.pem_cert.clone()]) } @@ -54,7 +92,7 @@ impl CertRequestClient { pub async fn create( keys: &AppKeys, - pccs_url: Option<&str>, + attestation_verifier: Arc<AttestationVerifier>, vm_config: String, ) -> Result<CertRequestClient> { match &keys.key_provider { @@ -79,7 +117,7 @@ impl CertRequestClient { .tls_client_key(client_cert.key_pem) .tls_ca_cert(keys.ca_cert.clone()) .tls_built_in_root_certs(false) - .maybe_pccs_url(pccs_url.map(|s| s.to_string())) + .attestation_verifier(attestation_verifier) .build() .into_client() .context("Failed to create RA client")?; @@ -89,3 +127,169 @@ impl CertRequestClient { } } } + +#[cfg(test)] +mod tests { + use super::*; + use ra_tls::{ + attestation::{Attestation, AttestationQuote, StackEvidence, TdxQuote}, + cert::{CertConfigV2, CertSigningRequestV2}, + rcgen::{BasicConstraints, CertificateParams, IsCa, KeyPair}, + traits::CertExt, + }; + + /// The guest agent simulator's attestation, the same bytes every gateway + /// suite runs against. Used rather than a hand-built one because the value + /// under test is what `decode_app_info` reads out of a real event log -- + /// a synthetic attestation would prove the plumbing and not the decode. + const SIMULATOR_ATTESTATION: &[u8] = include_bytes!("../../../sdk/simulator/attestation.bin"); + + /// The app id `SIMULATOR_ATTESTATION` decodes to. Written out rather than + /// recomputed, so a change in either the fixture or the decode shows up + /// here as a failure instead of being silently agreed with. + const SIMULATOR_APP_ID: &str = "5bb4ff9a3837357f19dc176407a5709c62eb6c56"; + + fn simulator_attestation() -> VersionedAttestation { + VersionedAttestation::from_bytes(SIMULATOR_ATTESTATION).expect("decode fixture") + } + + /// The simulator's attestation with the app-id event's payload emptied. + /// + /// This is the case the `.filter()` in `local_ca_app_id` exists for and the + /// only one that reaches it: `find_event_payload` returns an empty vec for a + /// missing or empty payload, so `decode_app_info` SUCCEEDS and hands back an + /// AppInfo whose app_id is empty. The undecodable attestation below does not + /// reach the filter at all -- it fails earlier, at `.ok()` -- so it cannot + /// stand in for this. Only the digest is measured, so clearing the payload + /// leaves the RTMR replay intact. + fn attestation_with_empty_app_id() -> VersionedAttestation { + let mut attestation = simulator_attestation().into_v1(); + let StackEvidence::Dstack { + ref mut runtime_events, + .. + } = attestation.stack + else { + panic!("the simulator fixture is expected to carry dstack stack evidence"); + }; + let mut found = false; + for event in runtime_events.iter_mut() { + if event.event == "app-id" { + event.payload = Vec::new(); + found = true; + } + } + assert!( + found, + "fixture must carry an app-id event for this to mean anything" + ); + VersionedAttestation::V1 { attestation } + } + + /// An attestation carrying nothing decodable: the case where no app id can + /// be read at all. + fn undecodable_attestation() -> VersionedAttestation { + Attestation { + quote: AttestationQuote::DstackTdx(TdxQuote { + quote: vec![], + event_log: vec![], + }), + runtime_events: vec![], + report_data: [0u8; 64], + config: "".into(), + report: (), + } + .into_versioned() + } + + fn csr_with(attestation: VersionedAttestation, pubkey: Vec<u8>) -> CertSigningRequestV2 { + CertSigningRequestV2 { + confirm: "please sign cert:".to_string(), + pubkey, + config: CertConfigV2 { + org_name: None, + subject: "local-ca-test".to_string(), + subject_alt_names: vec![], + usage_server_auth: true, + usage_client_auth: true, + ext_quote: false, + // Deliberately off. `sign_csr` derives app *info* separately + // when this is set, and hard fails when it cannot -- which + // would mask the app *id* path this is about. The gateway does + // set it, which is why the break showed on the client side + // only; see `local_ca_app_id`. + ext_app_info: false, + not_before: None, + not_after: None, + }, + attestation, + } + } + + fn local_client() -> CertRequestClient { + let key = KeyPair::generate().expect("ca key"); + let mut params = CertificateParams::new(vec![]).expect("ca params"); + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + let cert = params.self_signed(&key).expect("ca cert"); + let ca = CaCert::new(cert.pem(), key.serialize_pem()).expect("ca"); + CertRequestClient::Local { ca: Box::new(ca) } + } + + /// Sign through the real `CertRequestClient::Local` branch and read the + /// app-id extension back off the leaf, the way a peer does. + async fn issued_app_id(attestation: VersionedAttestation) -> Option<Vec<u8>> { + let leaf_key = KeyPair::generate().expect("leaf key"); + let csr = csr_with(attestation, leaf_key.public_key_der()); + let chain = local_client() + .sign_csr(&csr, &[]) + .await + .expect("local signing"); + let (_, leaf) = x509_parser::pem::parse_x509_pem(chain[0].as_bytes()).expect("leaf pem"); + let (_, leaf) = x509_parser::parse_x509_certificate(&leaf.contents).expect("leaf der"); + leaf.get_app_id().expect("read app id") + } + + /// The contract dstack-gateway's cluster mTLS rests on. + /// + /// `AppIdValidator` (the sync client) and `ensure_from_gateway` (the RPC + /// handler) read this extension with no fallback, so a locally issued + /// certificate without it is rejected by every peer. Reverting the stamp to + /// the `None` it used to pass turns this red. + #[tokio::test] + async fn a_locally_issued_certificate_carries_the_attested_app_id() { + let app_id = issued_app_id(simulator_attestation()).await; + assert_eq!( + app_id.as_deref().map(hex::encode).as_deref(), + Some(SIMULATOR_APP_ID), + "a peer that pins app_id must be able to read it off a local CA's certificate" + ); + } + + /// And what must NOT happen: an attestation yielding nothing leaves the + /// extension absent rather than stamping an empty value. + /// + /// An empty app id in the extension is not "unknown", it is a value, and + /// both consumers compare by equality -- so every app without an app-id + /// event would have matched every other one. Dropping the `.filter()` in + /// `local_ca_app_id` turns this red. + #[tokio::test] + async fn an_attestation_with_no_app_id_leaves_the_extension_absent() { + assert_eq!( + issued_app_id(attestation_with_empty_app_id()).await, + None, + "an empty app id must be absent, not stamped" + ); + assert_eq!(issued_app_id(undecodable_attestation()).await, None); + } + + /// The same rule at the unit it is decided in, so a failure says which of + /// the two halves moved. + #[test] + fn local_ca_app_id_is_absent_rather_than_empty() { + assert_eq!(local_ca_app_id(&attestation_with_empty_app_id()), None); + assert_eq!(local_ca_app_id(&undecodable_attestation()), None); + assert_eq!( + local_ca_app_id(&simulator_attestation()).map(hex::encode), + Some(SIMULATOR_APP_ID.to_string()) + ); + } +} diff --git a/dstack/certbot/Cargo.toml b/dstack/certbot/Cargo.toml index eaf594cff..ca06d9a95 100644 --- a/dstack/certbot/Cargo.toml +++ b/dstack/certbot/Cargo.toml @@ -31,5 +31,6 @@ x509-parser.workspace = true [dev-dependencies] rand.workspace = true +tempfile.workspace = true tokio = { workspace = true, features = ["full"] } tracing-subscriber.workspace = true diff --git a/dstack/certbot/cli/Cargo.toml b/dstack/certbot/cli/Cargo.toml index cd415c084..3698f7b85 100644 --- a/dstack/certbot/cli/Cargo.toml +++ b/dstack/certbot/cli/Cargo.toml @@ -23,5 +23,5 @@ serde.workspace = true tokio = { workspace = true, features = ["full"] } toml_edit.workspace = true tracing-subscriber.workspace = true -rustls.workspace = true +rustls = { workspace = true, features = ["ring"] } or-panic.workspace = true diff --git a/dstack/certbot/cli/README.md b/dstack/certbot/cli/README.md new file mode 100644 index 000000000..70a0240d9 --- /dev/null +++ b/dstack/certbot/cli/README.md @@ -0,0 +1,52 @@ +# certbot + +A small ACME client that issues and renews certificates over dns-01, with Cloudflare +as the DNS provider. It is a testing and development tool: it exercises the `certbot` +crate that dstack-gateway builds on, and it obtains a certificate by hand for a local +or staging setup. + +It is not the production path. A deployed gateway issues its own certificates from the +domain configuration and DNS credentials it already holds, publishes the resulting +public keys over `/acme-info` for `ct_monitor` to check the CT logs against, and — when +that gateway runs in a CVM — keeps both the ACME account key and the certificate key +inside the enclave. This CLI keeps them in a plain directory on whatever host runs it, +so nothing about the issuance is attested and the key is only as protected as that +filesystem. Point it at Pebble or Let's Encrypt staging while working on the ACME code, +not at the certificate fronting a real deployment. + +## Usage + +Write a configuration template, fill in the Cloudflare token and the names, then run +the daemon: + +```bash +certbot cfg --write-to certbot.toml +$EDITOR certbot.toml +RUST_LOG=info,certbot=debug certbot renew -c certbot.toml +``` + +`renew` issues a certificate if none is live, reissues when the live one does not carry +the configured `domains`, and renews once the live one is within `renew_days_before` of +expiry. It loops every `renew_interval` seconds; `--once` runs a single pass and exits, +and `--force` renews whether or not expiry is near. + +`init` creates the ACME account — and, with `auto_set_caa`, the CAA records — without +issuing anything. `set-caa` writes the CAA records for the configured names on their +own, which is worth running after adding a name to `domains`: the automatic pass only +happens when the account is created. + +## Configuration + +`certbot cfg` prints every field with its documentation. Two optional fields are absent +from that template because they default to nothing: + +- `cf_api_url` — a Cloudflare-compatible API base URL, for pointing the DNS calls at a + mock server instead of Cloudflare. +- `renewed_hook` — a shell command run after a certificate is committed, e.g. to reload + whatever is serving it. + +`workdir` holds everything else. Each issuance lands in its own timestamped directory +under `backup/`, `live/cert.pem` and `live/key.pem` are symlinks to the one in force, +and the ACME account credentials sit in `credentials.json`. Editing `domains` takes +effect on the next run: the live certificate's names are compared against the +configuration, and a mismatch reissues. diff --git a/dstack/certbot/cli/src/main.rs b/dstack/certbot/cli/src/main.rs index b30eade32..428f0ba6b 100644 --- a/dstack/certbot/cli/src/main.rs +++ b/dstack/certbot/cli/src/main.rs @@ -5,8 +5,8 @@ use std::{path::PathBuf, time::Duration}; -use anyhow::{Context, Result}; -use certbot::{CertBotConfig, WorkDir}; +use anyhow::{bail, Context, Result}; +use certbot::{CertBotConfig, ChallengeKind, WorkDir, LETS_ENCRYPT_ISSUER_DOMAIN_NAME}; use clap::Parser; use documented::DocumentedFields; use fs_err as fs; @@ -28,7 +28,7 @@ enum Command { #[arg(long)] force: bool, }, - /// Initialize the configuration file + /// Create the ACME account described by the configuration file Init { /// Path to the configuration file #[arg(short, long, default_value = "certbot.toml")] @@ -40,6 +40,15 @@ enum Command { #[arg(short, long, default_value = "certbot.toml")] config: PathBuf, }, + /// Print the DNS records the configured domains need + /// + /// With `challenge = "dns-persist-01"` these are not written by certbot and + /// have to be published once, by hand, before the first issuance. + DnsRecords { + /// Path to the configuration file + #[arg(short, long, default_value = "certbot.toml")] + config: PathBuf, + }, /// Generate configuration template Cfg { /// Write to file @@ -60,8 +69,30 @@ struct Config { workdir: PathBuf, /// ACME server URL acme_url: String, - /// Cloudflare API token + /// ACME challenge used to prove control of the domains + /// + /// "dns-01" (default) writes a TXT record per order through the Cloudflare + /// API and needs cf_api_token. + /// + /// "dns-persist-01" proves control with a _validation-persist TXT record + /// published once, by hand: no API token, and the zone can be hosted + /// anywhere. Run `certbot init` then `certbot dns-records` to get the + /// records to publish. Experimental: the draft is still changing and + /// Let's Encrypt serves this challenge on staging only. + #[serde(default)] + challenge: ChallengeKind, + /// Issuer Domain Name naming the CA in dns-persist-01 and CAA records + #[serde(default = "default_issuer_domain_name")] + issuer_domain_name: String, + /// Cloudflare API token (unused with dns-persist-01) + #[serde(default)] cf_api_token: String, + /// Optional Cloudflare-compatible API base URL + #[serde(default)] + cf_api_url: Option<String>, + /// TTL for DNS TXT challenge records in seconds + #[serde(default = "default_dns_txt_ttl")] + dns_txt_ttl: u32, /// Auto set CAA record auto_set_caa: bool, /// List of domains to issue certificates for @@ -84,7 +115,11 @@ impl Default for Config { Self { workdir: ".".into(), acme_url: "https://acme-staging-v02.api.letsencrypt.org/directory".into(), + challenge: ChallengeKind::default(), + issuer_domain_name: default_issuer_domain_name(), cf_api_token: "".into(), + cf_api_url: None, + dns_txt_ttl: default_dns_txt_ttl(), auto_set_caa: true, domains: vec!["example.com".into()], renew_interval: 3600, @@ -96,13 +131,28 @@ impl Default for Config { } } +const fn default_dns_txt_ttl() -> u32 { + 60 +} + +fn default_issuer_domain_name() -> String { + LETS_ENCRYPT_ISSUER_DOMAIN_NAME.to_string() +} + impl Config { fn to_commented_toml(&self) -> Result<String> { let mut doc = to_document(self)?; - for (i, (mut key, _value)) in doc.iter_mut().enumerate() { + for (mut key, _value) in doc.iter_mut() { + // Look the doc comment up by name rather than by position: a `None` + // option serializes to nothing, so the document's keys are a subset + // of the struct's fields and indexing `FIELD_DOCS` positionally + // attaches every comment after the first absent key to the wrong + // one. + let Ok(docstring) = Self::get_field_docs(key.get()) else { + continue; + }; let decor = key.leaf_decor_mut(); - let docstring = Self::FIELD_DOCS[i]; let mut comment = String::new(); for line in docstring.lines() { @@ -133,7 +183,11 @@ fn load_config(config: &PathBuf) -> Result<CertBotConfig> { .key_file(workdir.key_path()) .auto_create_account(true) .cert_subject_alt_names(config.domains) + .challenge(config.challenge) + .issuer_domain_name(config.issuer_domain_name) .cf_api_token(config.cf_api_token) + .maybe_cf_api_url(config.cf_api_url) + .dns_txt_ttl(config.dns_txt_ttl) .renew_interval(renew_interval) .renew_timeout(renew_timeout) .renew_expires_in(renew_expires_in) @@ -147,15 +201,52 @@ fn load_config(config: &PathBuf) -> Result<CertBotConfig> { async fn renew(config: &PathBuf, once: bool, force: bool) -> Result<()> { let bot_config = load_config(config).context("Failed to load configuration")?; - let bot = bot_config - .build_bot() - .await - .context("Failed to build bot")?; if once { - bot.renew(force).await?; - } else { + let bot = bot_config + .build_bot() + .await + .context("Failed to build bot")?; + bot.renew_and_run_hook(force).await?; + return Ok(()); + } + + // Startup and the renew loop are raced against shutdown as one future, so + // the signal listener covers the whole daemon lifetime. build_bot() does + // network I/O (resolving the DNS zone, creating the ACME account), and a + // SIGTERM/Ctrl-C arriving there used to fall through to the default + // disposition and hard-kill the process. Racing the two phases separately + // would leave a gap between them with no listener registered: tokio never + // restores the default disposition once a handler is installed, and drops + // a signal that arrives while nothing is subscribed, so such a SIGTERM + // would neither terminate nor unblock the process. + let daemon = async { + let bot = bot_config + .build_bot() + .await + .context("Failed to build bot")?; bot.run().await; + // run() loops forever today, but that is not enforced by its type: a + // future early return should exit with an error, not panic. + bail!("certbot daemon exited unexpectedly") + }; + tokio::select! { + result = daemon => result, + result = shutdown_signal() => result, + } +} + +async fn shutdown_signal() -> Result<()> { + #[cfg(unix)] + { + let mut terminate = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + tokio::select! { + result = tokio::signal::ctrl_c() => result?, + _ = terminate.recv() => {}, + } } + #[cfg(not(unix))] + tokio::signal::ctrl_c().await?; Ok(()) } @@ -192,6 +283,16 @@ async fn main() -> Result<()> { .context("Failed to build bot")?; bot.set_caa().await?; } + Command::DnsRecords { config } => { + let bot_config = load_config(&config).context("Failed to load configuration")?; + let bot = bot_config + .build_bot() + .await + .context("Failed to build bot")?; + for record in bot.required_dns_records() { + println!("{record}"); + } + } Command::Cfg { write_to } => { let toml_str = Config::default().to_commented_toml()?; match write_to { @@ -202,3 +303,62 @@ async fn main() -> Result<()> { } Ok(()) } + +#[cfg(test)] +mod config_template_tests { + use super::*; + + /// Every key has to carry its own doc comment. `toml_edit` drops `None` + /// fields from the serialized document -- `cf_api_url` is unset by default + /// -- so indexing `FIELD_DOCS` by position shifts every comment after the + /// gap onto the wrong setting. + #[test] + fn every_key_is_labelled_with_its_own_doc_comment() { + let rendered = Config::default() + .to_commented_toml() + .expect("the default config renders"); + + // Pair each key with the comment block immediately above it. + let mut comment = String::new(); + let mut pairs = Vec::new(); + for line in rendered.lines() { + match line.strip_prefix('#') { + Some(text) => comment.push_str(text.trim()), + None => { + if let Some((key, _)) = line.split_once('=') { + pairs.push((key.trim().to_string(), std::mem::take(&mut comment))); + } + } + } + } + assert!(!pairs.is_empty(), "no keys rendered:\n{rendered}"); + + for (key, comment) in &pairs { + let expected = Config::get_field_docs(key) + .unwrap_or_else(|err| panic!("no doc comment for {key:?}: {err}")); + let expected: String = expected.lines().map(str::trim).collect(); + assert_eq!( + comment, &expected, + "key {key:?} is labelled with another field's doc comment" + ); + } + } + + /// The field that made the misalignment visible: it sat after the dropped + /// `cf_api_url` and was labelled "Renew timeout in seconds", next to the one + /// value whose interaction with `renew_timeout` this crate clamps. + #[test] + fn max_dns_wait_is_not_labelled_as_a_renew_timeout() { + let rendered = Config::default() + .to_commented_toml() + .expect("the default config renders"); + let (before, _) = rendered + .split_once("max_dns_wait") + .expect("max_dns_wait is rendered"); + let label = before.lines().last().expect("it has a comment above it"); + assert!( + label.contains("DNS propagation"), + "max_dns_wait is labelled {label:?}" + ); + } +} diff --git a/dstack/certbot/src/acme_client.rs b/dstack/certbot/src/acme_client.rs index 60b0dec12..efa1652bc 100644 --- a/dstack/certbot/src/acme_client.rs +++ b/dstack/certbot/src/acme_client.rs @@ -4,42 +4,346 @@ use anyhow::{bail, Context, Result}; use fs_err as fs; +use hickory_resolver::config::{NameServerConfig, ResolverConfig}; +use hickory_resolver::net::runtime::TokioRuntimeProvider; use hickory_resolver::proto::rr::RData; use hickory_resolver::TokioResolver; use instant_acme::{ - Account, AccountCredentials, AuthorizationStatus, ChallengeType, Identifier, NewAccount, - NewOrder, Order, OrderStatus, Problem, + Account, AccountCredentials, AuthorizationStatus, AuthorizedIdentifier, ChallengeType, + Identifier, NewAccount, NewOrder, Order, OrderStatus, Problem, }; use rcgen::{CertificateParams, DistinguishedName, KeyPair}; use serde::{Deserialize, Serialize}; use std::{ - collections::BTreeSet, + collections::{BTreeMap, BTreeSet}, + fmt, + net::SocketAddr, path::{Path, PathBuf}, time::Duration, }; use tokio::time::sleep; -use tracing::{debug, error, info}; +use tracing::{debug, error, info, warn}; use x509_parser::prelude::{GeneralName, Pem}; use super::dns01_client::{Dns01Api, Dns01Client}; +use super::dns_persist::{self, AuthorizationRecord}; use super::http_client::ReqwestHttpClient; +/// The ACME challenge used to prove control of the certificate's domains. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum ChallengeKind { + /// RFC 8555 `dns-01`. certbot answers every order by writing a TXT record + /// through the DNS provider API, so it needs a credential with write access + /// to the zone. + #[default] + #[serde(rename = "dns-01")] + Dns01, + /// draft-ietf-acme-dns-persist-01 `dns-persist-01`. Control is proven by a + /// `_validation-persist` record published once, out of band; certbot needs no + /// DNS credential and the zone can be hosted anywhere. + /// + /// Experimental — see `docs/certbot-dns-persist-01.md`. + #[serde(rename = "dns-persist-01")] + DnsPersist01, +} + +/// How the client proves control of a domain to the ACME server. +/// +/// The two methods differ in who writes DNS. `dns-01` needs certbot to hold a +/// provider credential with write access to the zone for the lifetime of the +/// deployment; `dns-persist-01` moves that to a one-time record the operator +/// publishes by hand, after which certbot only ever reads DNS. +#[derive(Debug)] +pub enum ValidationMethod { + /// RFC 8555 `dns-01`: certbot publishes a fresh `_acme-challenge` TXT record + /// through the provider API for every order and removes it afterwards. + Dns01 { + /// Provider client with write access to the zone. + client: Dns01Client, + /// TTL of the published records, in seconds (1 = auto, min 60 on Cloudflare). + txt_ttl: u32, + /// Issuer Domain Name to name in the CAA records certbot publishes. + issuer_domain_name: String, + }, + /// draft-ietf-acme-dns-persist-01 `dns-persist-01`: control is proven by a + /// `_validation-persist` TXT record naming the CA and this ACME account, + /// published once and left in place. certbot needs no provider credential, + /// and the zone can be hosted anywhere. + /// + /// Experimental: the draft is still changing and Let's Encrypt serves this + /// challenge on staging only. See `docs/certbot-dns-persist-01.md`. + DnsPersist01 { + /// Issuer Domain Name to name in the record and in CAA records. Must be + /// one of the `issuer-domain-names` the CA sends in the challenge — + /// `letsencrypt.org` for Let's Encrypt. + issuer_domain_name: String, + }, +} + +impl ValidationMethod { + /// Which challenge this method answers. + fn kind(&self) -> ChallengeKind { + match self { + Self::Dns01 { .. } => ChallengeKind::Dns01, + Self::DnsPersist01 { .. } => ChallengeKind::DnsPersist01, + } + } + + /// The challenge type to look for in an authorization. + fn challenge_type(&self) -> ChallengeType { + match self { + Self::Dns01 { .. } => ChallengeType::Dns01, + // instant-acme has no variant for the draft challenge, so it lands in + // `Unknown`. Matching on the wire string is what selects it. + Self::DnsPersist01 { .. } => ChallengeType::Unknown(DNS_PERSIST_01.to_string()), + } + } + + /// Issuer Domain Name to write into CAA records. + /// + /// Both methods read it from configuration, whose default is + /// `letsencrypt.org` -- the name existing deployments already have published + /// -- so an untouched configuration writes what it wrote before. A CAA + /// record naming a CA other than the one at `acme_url` forbids the very + /// issuance it is published to enable, and that is not a dns-01/ + /// dns-persist-01 distinction. + fn issuer_domain_name(&self) -> &str { + match self { + Self::Dns01 { + issuer_domain_name, .. + } => issuer_domain_name, + Self::DnsPersist01 { issuer_domain_name } => issuer_domain_name, + } + } + + /// The provider client, or an error naming why there isn't one. + fn dns01_client(&self) -> Result<&Dns01Client> { + match self { + Self::Dns01 { client, .. } => Ok(client), + Self::DnsPersist01 { .. } => bail!( + "dns-persist-01 holds no DNS provider credential, so certbot cannot write \ + records; publish them out of band (see the `dns-records` command)" + ), + } + } +} + +/// Wire name of the draft challenge, as it appears in the authorization. +const DNS_PERSIST_01: &str = "dns-persist-01"; + +/// A DNS record that has to exist before the CA will issue. +/// +/// Rendered as a zone-file line so it can be pasted into any provider. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RequiredRecord { + /// FQDN the record lives at. + pub name: String, + /// Record type, e.g. `TXT` or `CAA`. + pub record_type: String, + /// Record value, including any CAA flags and tag. + pub content: String, +} + +impl fmt::Display for RequiredRecord { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}. IN {} {}", self.name, self.record_type, self.content) + } +} + +/// The CAA `issue`/`issuewild` value that pins issuance to one account. +/// +/// `validationmethods` names the challenge actually in use: a record left +/// pinned to `dns-01` after switching to `dns-persist-01` refuses every order, +/// and vice versa. +fn caa_content(challenge: ChallengeKind, issuer_domain_name: &str, account_uri: &str) -> String { + let method = match challenge { + ChallengeKind::Dns01 => "dns-01", + ChallengeKind::DnsPersist01 => DNS_PERSIST_01, + }; + format!("{issuer_domain_name};validationmethods={method};accounturi={account_uri}") +} + +/// Every DNS record that has to exist for the CA to issue for `domains`. +/// +/// Under `dns-01` certbot writes these itself and the list is informational. +/// Under `dns-persist-01` it holds no credential and cannot write anything, so +/// this list *is* the one-time setup an operator has to publish by hand. +/// +/// Takes the account URI rather than a client so that callers holding only the +/// stored credentials — an admin listing, say — can render the records without +/// a round trip to the CA. +pub fn required_dns_records( + challenge: ChallengeKind, + issuer_domain_name: &str, + account_uri: &str, + domains: &[String], +) -> Vec<RequiredRecord> { + let caa_content = caa_content(challenge, issuer_domain_name, account_uri); + let mut records = Vec::new(); + for base_name in base_names(domains) { + if challenge == ChallengeKind::DnsPersist01 { + // One record covers the base name and, with the wildcard policy, + // `*.<base name>`; only ask for the policy when a wildcard is + // actually requested, so the record grants no more than needed. + let record = AuthorizationRecord { + issuer_domain_name: issuer_domain_name.to_string(), + account_uri: account_uri.to_string(), + wildcard: domains + .iter() + .any(|name| name.strip_prefix("*.") == Some(base_name)), + }; + records.push(RequiredRecord { + name: dns_persist::validation_domain(base_name), + record_type: "TXT".to_string(), + content: format!("\"{}\"", record.rdata()), + }); + } + for tag in ["issue", "issuewild"] { + records.push(RequiredRecord { + name: base_name.to_string(), + record_type: "CAA".to_string(), + content: format!("0 {tag} \"{caa_content}\""), + }); + } + } + records +} + +/// An ACME account, as registered, before any validation method is chosen. +/// +/// Returned by [`AcmeClient::register_account`] so a caller can store the +/// credentials and report the URI -- the two things a `dns-persist-01` record +/// and an account-pinned CAA record are written from -- without holding a +/// client it has no use for yet. +pub struct AcmeAccount { + /// Encoded credentials, in the form [`AcmeClient::load`] takes. + pub credentials: String, + /// URI the CA identifies this account by. + pub account_uri: String, +} + +/// The share of `renew_timeout` the pre-order DNS check may spend. +/// +/// The rest pays for the order itself: `new_order`, the authorizations, the +/// provider writes under `dns-01`, then finalize and the certificate fetch. +const DNS_WAIT_SHARE_OF_RENEW_TIMEOUT: u32 = 2; + +/// The DNS wait to actually use, given what is configured and how long the whole +/// order is allowed to take. +/// +/// The wait is advisory by design: certbot polls its own resolver, logs what it +/// could not see, and starts the order regardless, because the CA's DNS view is +/// not this node's. That only holds if the wait ends before the timeout wrapping +/// the order does. It does not by default -- a DNS credential's `max_dns_wait` +/// defaults to 300s and `renew_timeout` defaults to 300s, and the wait is +/// measured from after the order and its authorizations are fetched, so the +/// outer timeout always fires first. The renewal then dies with "certificate +/// request timed out", and the warning naming the record that was missing -- +/// the first thing an operator is told to look for -- is never logged. +/// +/// Clamping here rather than picking a smaller constant covers both challenges +/// with one rule, and keeps holding when an operator lowers `renew_timeout` +/// from the dashboard, which no constant can. +pub fn advisory_dns_wait(configured: Duration, renew_timeout: Duration) -> Duration { + let capped = renew_timeout / DNS_WAIT_SHARE_OF_RENEW_TIMEOUT; + if configured > capped { + // At `debug!` deliberately: the stock defaults have both values at 300s, + // so this fires on every issuance in every deployment. Phrased as the + // budget it is rather than as an override, so an operator who does raise + // `max_dns_wait` and finds nothing changed can see why here, without a + // permanent line in everyone else's log implying they configured + // something that was ignored. + debug!( + "DNS check budget is {capped:?}, half of the {renew_timeout:?} renewal timeout; \ + the configured wait of {configured:?} does not fit inside it" + ); + } + configured.min(capped) +} + +/// How long the DNS poll may sleep before its next lookup. +/// +/// The backoff, cut to whatever is left of the budget. Split out from the loop +/// because the loop cannot be driven from a test -- it needs a resolver and a +/// live challenge -- while the arithmetic is the whole of the rule and is what +/// went wrong: sleeping the full backoff and checking the budget afterwards +/// overshoots by up to a whole step (32s at the top of the ramp), which for a +/// short `renew_timeout` hands the deadline to the timeout wrapping the order. +/// The graceful "proceed anyway" exit is then missed and the renewal ends as a +/// bare timeout, naming nothing. +/// +/// A zero budget yields a zero sleep, and the caller's expiry check fires on the +/// same pass, so `max_dns_wait = 0` means the check is skipped outright rather +/// than run once. +fn dns_poll_sleep(budget: Duration, elapsed: Duration, backoff: Duration) -> Duration { + budget.saturating_sub(elapsed).min(backoff) +} + /// A AcmeClient instance. pub struct AcmeClient { account: Account, credentials: Credentials, - dns01_client: Dns01Client, + validation: ValidationMethod, max_dns_wait: Duration, - /// TTL for DNS TXT records used in ACME challenges (in seconds). - dns_txt_ttl: u32, } +/// One pending authorization and the DNS record that answers it. #[derive(Debug, Clone)] struct Challenge { - id: String, + /// Provider-assigned record id, used by the cleanup pass after the order + /// settles. `None` when certbot did not create the record and must not + /// delete it — `dns-persist-01` records belong to the operator. + id: Option<String>, + /// FQDN the TXT record lives at. acme_domain: String, - url: String, - dns_value: String, + /// What a TXT record there has to say for the CA to accept the challenge. + expected: Expected, +} + +/// The condition a challenge's TXT records have to meet. +#[derive(Debug, Clone)] +enum Expected { + /// `dns-01`: the key authorization digest, matched verbatim. + KeyAuthorization(String), + /// `dns-persist-01`: an issue-value naming our issuer and account. + Authorization(AuthorizationRecord), +} + +impl Expected { + /// Whether the TXT records currently published at the challenge domain answer + /// the challenge. + fn satisfied_by(&self, published: &[String], now: u64) -> bool { + match self { + Self::KeyAuthorization(value) => published.iter().any(|txt| txt == value), + Self::Authorization(record) => record.satisfied_by_any(published, now), + } + } +} + +impl Challenge { + /// A line naming what is missing at the challenge domain, phrased so an + /// operator can act on it: under `dns-persist-01` it is the exact record to + /// publish, under `dns-01` it is the value certbot just wrote. + fn unsettled_hint(&self) -> String { + format!( + "no TXT record at {} matches the expected value: {}", + self.acme_domain, self.expected + ) + } +} + +impl fmt::Display for Expected { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::KeyAuthorization(value) => f.write_str(value), + Self::Authorization(record) => write!(f, "{}", record.rdata()), + } + } +} + +/// Current UNIX time, for comparing against a record's `persistUntil`. +fn now_secs() -> u64 { + time::OffsetDateTime::now_utc().unix_timestamp().max(0) as u64 } #[derive(Serialize, Deserialize)] @@ -57,61 +361,117 @@ pub(crate) fn acme_matches(encoded_credentials: &str, acme_url: &str) -> bool { credentials.acme_url == acme_url } +/// The names CAA and `dns-persist-01` records are published under, deduplicated. +/// +/// A wildcard request is authorized from its base name, so `example.com` and +/// `*.example.com` share one set of records. +fn base_names(domains: &[String]) -> BTreeSet<&str> { + domains + .iter() + .map(|name| name.strip_prefix("*.").unwrap_or(name)) + .collect() +} + +fn caa_tag(content: &str) -> Option<&str> { + content.split_whitespace().nth(1) +} + +/// Whether a rendered CAA record is one of the `;` guards [`AcmeClient::set_caa_records`] +/// installs while it swaps the real records out. +/// +/// A guard denies every issuer, which is the point while the old records are +/// being replaced and the wrong thing to leave behind. +fn is_caa_guard(content: &str) -> bool { + let mut parts = content.split_whitespace(); + let (_flags, tag, value) = (parts.next(), parts.next(), parts.next()); + matches!(tag, Some("issue" | "issuewild")) + && value.map(|value| value.trim_matches('"')) == Some(";") + && parts.next().is_none() +} + impl AcmeClient { pub async fn load( - dns01_client: Dns01Client, + validation: ValidationMethod, encoded_credentials: &str, max_dns_wait: Duration, - dns_txt_ttl: u32, ) -> Result<Self> { let credentials: Credentials = serde_json::from_str(encoded_credentials)?; let http_client = Box::new(ReqwestHttpClient::new()?); - let account = - Account::from_credentials_and_http(credentials.credentials, http_client).await?; + let account = Account::builder_with_http(http_client) + .from_credentials(credentials.credentials) + .await?; let credentials: Credentials = serde_json::from_str(encoded_credentials)?; Ok(Self { account, - dns01_client, + validation, credentials, max_dns_wait, - dns_txt_ttl, }) } - /// Create a new account. + /// Register a new ACME account, with no validation method attached. + /// + /// Registration is a POST to the directory's `newAccount`. It asserts + /// nothing about any domain and touches no DNS, so it needs neither a + /// provider credential nor a challenge choice -- and a deployment can + /// therefore have an account before it has anything to validate. That order + /// is forced under `dns-persist-01`: the record an operator publishes names + /// the account, so the account has to exist first. + pub async fn register_account(acme_url: &str) -> Result<AcmeAccount> { + let (_, credentials) = Self::register(acme_url).await?; + Ok(AcmeAccount { + account_uri: credentials.account_id.clone(), + credentials: serde_json::to_string(&credentials)?, + }) + } + + /// Create a new account and a client bound to `validation`. pub async fn new_account( acme_url: &str, - dns01_client: Dns01Client, + validation: ValidationMethod, max_dns_wait: Duration, - dns_txt_ttl: u32, ) -> Result<Self> { - let http_client = Box::new(ReqwestHttpClient::new()?); - let (account, credentials) = Account::create_with_http( - &NewAccount { - contact: &[], - terms_of_service_agreed: true, - only_return_existing: false, - }, - acme_url, - None, - http_client, - ) - .await - .with_context(|| format!("failed to create ACME account for {acme_url}"))?; - let credentials = Credentials { - acme_url: acme_url.to_string(), - account_id: account.id().to_string(), - credentials, - }; + // Built from the account `register` already holds rather than by + // reloading the credentials: `load` fetches the directory, and a + // transient failure there would drop an account that exists at the CA + // and counts against its registration limit, on a path whose caller has + // not persisted anything yet. + let (account, credentials) = Self::register(acme_url).await?; Ok(Self { account, - dns01_client, + validation, credentials, max_dns_wait, - dns_txt_ttl, }) } + /// Register an account and keep both halves: the live account and the + /// credentials that reconstruct it. + async fn register(acme_url: &str) -> Result<(Account, Credentials)> { + let http_client = Box::new(ReqwestHttpClient::new()?); + let (account, credentials) = Account::builder_with_http(http_client) + .create( + &NewAccount { + contact: &[], + terms_of_service_agreed: true, + only_return_existing: false, + }, + acme_url.to_string(), + None, + ) + .await + .with_context(|| format!("failed to create ACME account for {acme_url}"))?; + let account_id = account.id().to_string(); + Ok(( + account, + Credentials { + acme_url: acme_url.to_string(), + account_id, + credentials, + }, + )) + } + /// Dump the account credentials to a JSON string. pub fn dump_credentials(&self) -> Result<String> { Ok(serde_json::to_string(&self.credentials)?) @@ -122,51 +482,116 @@ impl AcmeClient { &self.credentials.account_id } + /// The CAA `issue`/`issuewild` value that pins issuance to this account. + fn caa_content(&self) -> String { + caa_content( + self.validation.kind(), + self.validation.issuer_domain_name(), + self.account_id(), + ) + } + + /// Every DNS record that has to exist for the CA to issue for `domains`. + /// + /// See [`required_dns_records`], which this fills in from the live account. + pub fn required_dns_records(&self, domains: &[String]) -> Vec<RequiredRecord> { + required_dns_records( + self.validation.kind(), + self.validation.issuer_domain_name(), + self.account_id(), + domains, + ) + } + pub async fn set_caa_records(&self, domains: &[String]) -> Result<()> { - let account_id = self.account_id(); - let content = format!("letsencrypt.org;validationmethods=dns-01;accounturi={account_id}"); - let base_names = domains - .iter() - .map(|name| name.strip_prefix("*.").unwrap_or(name)) - .collect::<BTreeSet<_>>(); + let dns01_client = self.validation.dns01_client().context( + "cannot set CAA records without DNS write access; publish the records \ + printed by `certbot dns-records` instead", + )?; + let content = self.caa_content(); + let base_names = base_names(domains); for base_name in base_names { + // 0. Adopt guards an earlier run left behind, rather than + // replacing them. A run that died between steps 1 and 4 stranded + // `;` records, and step 1 would re-add byte-identical ones -- + // which a provider that refuses duplicates rejects, so the rerun + // the operator is told to perform fails at its very first call + // and the zone never recovers. + // + // Deleting them first would fix that and open a worse hole: from + // the delete until step 1 succeeds the name has no issue or + // issuewild record at all, which is not "denied" but "any CA may + // issue" -- and a certificate obtained in that window outlives + // the window, where a stranded guard is merely a renewal that + // fails until someone reruns. Reusing the record keeps the + // deny-all continuous, which is the whole point of a guard. + // + // Extras beyond the two adopted here are ordinary issue records + // to step 2, which removes them. + let mut adopted_issue = None; + let mut adopted_issuewild = None; + for record in dns01_client.get_records(base_name).await? { + if record.r#type != "CAA" || !is_caa_guard(&record.content) { + continue; + } + let slot = match caa_tag(&record.content) { + Some("issue") => &mut adopted_issue, + Some("issuewild") => &mut adopted_issuewild, + _ => continue, + }; + if slot.is_none() { + debug!("adopting stale guard CAA record {}", record.name); + *slot = Some(record.id); + } + } // 1. Set ";" to guard timing gap between the operations. debug!("setting guard CAA records for {base_name}"); - let guard0 = self - .dns01_client - .add_caa_record(base_name, 0, "issue", ";") - .await?; - let guard1 = self - .dns01_client - .add_caa_record(base_name, 0, "issuewild", ";") - .await?; + let guard0 = match adopted_issue { + Some(id) => id, + None => { + dns01_client + .add_caa_record(base_name, 0, "issue", ";") + .await? + } + }; + let guard1 = match adopted_issuewild { + Some(id) => id, + None => { + dns01_client + .add_caa_record(base_name, 0, "issuewild", ";") + .await? + } + }; // 2. Remove the existing constraints - for record in self.dns01_client.get_records(base_name).await? { + for record in dns01_client.get_records(base_name).await? { if record.id == guard0 || record.id == guard1 { continue; } - if record.r#type == "CAA" { + if record.r#type == "CAA" + && caa_tag(&record.content) + .is_some_and(|tag| matches!(tag, "issue" | "issuewild")) + { debug!( - "removing existing CAA record {} {}", + "removing existing issuer CAA record {} {}", record.name, record.content ); - self.dns01_client.remove_record(&record.id).await?; + dns01_client.remove_record(&record.id).await?; } } // 3. Set the new constraints debug!("setting CAA records for {base_name}, 0 issue \"{content}\""); - self.dns01_client + dns01_client .add_caa_record(base_name, 0, "issue", &content) .await?; debug!("setting CAA records for {base_name}, 0 issuewild \"{content}\""); - self.dns01_client + dns01_client .add_caa_record(base_name, 0, "issuewild", &content) .await?; debug!("removing guard CAA records for {base_name}"); // 4. Remove the guards - self.dns01_client.remove_record(&guard0).await?; - self.dns01_client.remove_record(&guard1).await?; + dns01_client.remove_record(&guard0).await?; + dns01_client.remove_record(&guard1).await?; } Ok(()) } @@ -180,10 +605,18 @@ impl AcmeClient { let result = self .request_new_certificate_inner(key, domains, &mut challenges) .await; - for challenge in &challenges { - debug!("removing dns record {}", challenge.id); - if let Err(err) = self.dns01_client.remove_record(&challenge.id).await { - error!("failed to remove dns record {}: {err}", challenge.id); + // Only records certbot created are cleaned up. A dns-persist-01 record + // is the operator's and outlives every order, so it carries no id. + for id in challenges + .iter() + .filter_map(|challenge| challenge.id.as_ref()) + { + let Ok(dns01_client) = self.validation.dns01_client() else { + break; + }; + debug!("removing dns record {id}"); + if let Err(err) = dns01_client.remove_record(id).await { + error!("failed to remove dns record {id}: {err}"); } } result @@ -280,7 +713,10 @@ impl AcmeClient { Ok(()) } - /// Auto renew given certificate + /// Issue a certificate for `domains` unless the live one is already for + /// exactly those names. + /// + /// Returns whether a certificate was issued. pub async fn create_cert_if_needed( &self, domains: &[String], @@ -289,7 +725,28 @@ impl AcmeClient { backup_dir: impl AsRef<Path>, ) -> Result<bool> { if live_cert_pem_path.as_ref().exists() && live_key_pem_path.as_ref().exists() { - return Ok(false); + // The live certificate is only "the certificate we were asked for" + // if it carries the configured names. Treating its mere existence as + // sufficient pinned the name list at whatever the first issuance + // used: an operator adding a name to the configuration got no new + // certificate, and renewal read its names back off the old one, so + // the edit never took effect at all. + let reason = match fs::read_to_string(live_cert_pem_path.as_ref()) { + Ok(live_cert_pem) => reissue_reason(&live_cert_pem, domains), + // `exists()` has already passed, so this is a permission + // problem or a file replaced mid-flight rather than a missing + // certificate. Either way the certificate cannot be checked + // against the configuration, which is a reissue for the same + // reason an unparseable one is. + Err(err) => Some(format!( + "cannot read {}: {err:#}", + live_cert_pem_path.as_ref().display() + )), + }; + match reason { + None => return Ok(false), + Some(reason) => info!("reissuing: {reason}"), + } } let key_pem = if live_key_pem_path.as_ref().exists() { debug!("using existing cert key pair"); @@ -313,56 +770,139 @@ impl AcmeClient { impl AcmeClient { async fn authorize(&self, order: &mut Order, challenges: &mut Vec<Challenge>) -> Result<()> { - let authorizations = order - .authorizations() - .await - .context("failed to get authorizations")?; - for authz in &authorizations { + let challenge_type = self.validation.challenge_type(); + let mut authorizations = order.authorizations(); + while let Some(authz) = authorizations.next().await { + let mut authz = authz.context("failed to get authorizations")?; match authz.status { AuthorizationStatus::Pending => {} AuthorizationStatus::Valid => continue, _ => bail!("unsupported authorization status: {:?}", authz.status), } + // Read before taking the challenge handle, which borrows the + // authorization for the rest of the iteration. + let wildcard = authz.wildcard; let challenge = authz - .challenges - .iter() - .find(|c| c.r#type == ChallengeType::Dns01) - .context("no dns01 challenge found")?; - - let Identifier::Dns(identifier) = &authz.identifier; - - let dns_value = order.key_authorization(challenge).dns_value(); - debug!("creating dns record for {identifier}"); - let acme_domain = format!("_acme-challenge.{identifier}"); - debug!("removing existing TXT record for {acme_domain}"); - self.dns01_client - .remove_txt_records(&acme_domain) - .await - .context("failed to remove existing dns record")?; - debug!( - "creating TXT record for {acme_domain} with TTL {}s", - self.dns_txt_ttl - ); - let id = self - .dns01_client - .add_txt_record(&acme_domain, &dns_value, self.dns_txt_ttl) - .await - .context("failed to create dns record")?; - challenges.push(Challenge { - id, - acme_domain, - url: challenge.url.clone(), - dns_value, - }); + .challenge(challenge_type.clone()) + .with_context(|| format!("no {challenge_type:?} challenge found"))?; + + let acme_domain = challenge_domain(self.validation.kind(), challenge.identifier())?; + let challenge = match &self.validation { + ValidationMethod::Dns01 { + client, txt_ttl, .. + } => { + let dns_value = challenge.key_authorization().dns_value(); + // Clearing stale records is a per-name preparation step, not a + // per-authorization one. An order for `example.com` and + // `*.example.com` yields two authorizations that are both answered + // under `_acme-challenge.example.com`, each with its own value, and + // both values have to be live at validation time. Purging again for + // the second authorization would delete the record the first one + // just published, so one of the two challenges could never be + // answered and the order failed with "Correct value not found for + // DNS challenge". + if needs_purge(challenges, &acme_domain) { + debug!("removing existing TXT records for {acme_domain}"); + client + .remove_txt_records(&acme_domain) + .await + .context("failed to remove existing dns record")?; + } + debug!("creating TXT record for {acme_domain} with TTL {txt_ttl}s"); + let id = client + .add_txt_record(&acme_domain, &dns_value, *txt_ttl) + .await + .context("failed to create dns record")?; + Challenge { + id: Some(id), + acme_domain, + expected: Expected::KeyAuthorization(dns_value), + } + } + // Nothing to publish: the record is already in the zone, or the + // order is about to fail and say so. There is likewise nothing + // to purge -- the record is the operator's, and one persistent + // record answers every authorization under the name. + ValidationMethod::DnsPersist01 { issuer_domain_name } => Challenge { + id: None, + acme_domain, + expected: Expected::Authorization(AuthorizationRecord { + issuer_domain_name: issuer_domain_name.clone(), + account_uri: self.account_id().to_string(), + wildcard, + }), + }, + }; + challenges.push(challenge); } Ok(()) } - /// Self check the TXT records for the given challenges. - async fn check_dns(&self, challenges: &[Challenge]) -> Result<()> { - use tracing::warn; + /// Build a resolver that talks straight to the authoritative nameservers for + /// the challenge zone. + /// + /// The self-check runs moments after the TXT record is created, so the first + /// lookup can race propagation and come back NXDOMAIN. A recursive resolver + /// then caches that negative for the zone's SOA minimum -- 1800s is common, + /// far longer than `max_dns_wait` -- and every retry inside the window is + /// answered from that cache, so the check can never pass no matter how long + /// it waits. Reading from the authoritative servers removes the cache from + /// the path entirely. + async fn authoritative_resolver(&self, domain: &str) -> Result<TokioResolver> { + // The NS records and the nameservers' own addresses are stable, so the + // system resolver -- caching and all -- is the right tool for finding + // them. Only the challenge record itself must dodge the cache. + let bootstrap = system_resolver()?; + // `_acme-challenge.<name>` is almost never a zone cut, and neither is + // the name below it: for `_acme-challenge.a.example.com` the NS records + // usually live on `example.com`. Querying the full name returns NODATA, + // so walk up a label at a time until a name actually carries NS records. + // The leading label is dropped for any challenge prefix -- `_acme-challenge` + // for dns-01, `_validation-persist` for dns-persist-01 -- since an + // underscore label never carries NS records. + let mut candidate = strip_challenge_label(domain).to_string(); + let mut addrs = Vec::new(); + let mut zone = candidate.clone(); + loop { + if let Ok(ns_lookup) = bootstrap.ns_lookup(&candidate).await { + for answer in ns_lookup.answers() { + let RData::NS(ns) = &answer.data else { + continue; + }; + let Ok(ips) = bootstrap.lookup_ip(ns.0.to_utf8()).await else { + continue; + }; + addrs.extend(ips.iter().map(|ip| SocketAddr::new(ip, 53))); + } + if !addrs.is_empty() { + zone = candidate; + break; + } + } + match parent_zone(&candidate) { + Some(parent) => candidate = parent, + None => break, + } + } + if addrs.is_empty() { + bail!("no authoritative nameserver found for {domain}"); + } + addrs.sort(); + addrs.dedup(); + debug!("checking {domain} against authoritative nameservers for {zone}: {addrs:?}"); + resolver_for(&addrs) + } + + /// Wait until every challenge's records are visible to us. + /// + /// Advisory, not a gate: our resolver is not the CA's, and under + /// `dns-persist-01` our expectation can even be stricter than the CA's -- + /// the `issuer-domain-names` it would accept are not exposed by + /// instant-acme. A record we never see is reported and the order proceeds, + /// letting the CA decide. + async fn check_dns(&self, challenges: &[Challenge]) -> Result<()> { let mut delay = Duration::from_millis(250); let mut tries = 1u8; @@ -370,10 +910,47 @@ impl AcmeClient { debug!("Unsettled challenges: {unsettled_challenges:#?}"); + // The budget covers discovery as well as the wait. `renew_timeout` is + // commonly the same value as `max_dns_wait`, so leaving discovery + // outside it lets the outer timeout kill the renewal mid-wait instead of + // reaching the graceful "proceed anyway" exit below -- and that exit is + // what keeps a DNS problem from failing issuance outright. let start_time = std::time::Instant::now(); + // Resolve each challenge's nameservers once. A SAN list can span zones, + // so this is keyed per challenge rather than one resolver for all of + // them; and with caching disabled there is nothing to gain by repeating + // discovery on every retry. + let mut resolvers = BTreeMap::new(); + let mut fell_back = BTreeSet::new(); + for challenge in &unsettled_challenges { + if resolvers.contains_key(&challenge.acme_domain) { + continue; + } + let resolver = match self.authoritative_resolver(&challenge.acme_domain).await { + Ok(resolver) => resolver, + Err(err) => { + warn!( + "no authoritative nameserver for {} ({err:#}), using the system resolver", + challenge.acme_domain + ); + system_resolver()? + } + }; + resolvers.insert(challenge.acme_domain.clone(), resolver); + } + 'outer: loop { - sleep(delay).await; + // The budget is checked before sleeping, and the sleep is cut to + // what is left of it. Sleeping first and checking after overshoots + // by a whole backoff step -- up to 32s -- which for a short + // `renew_timeout` is enough to hand the deadline to the timeout + // wrapping the order, so the graceful exit below is missed and the + // renewal ends as a timeout with nothing named. + let nap = dns_poll_sleep(self.max_dns_wait, start_time.elapsed(), delay); + if !nap.is_zero() { + sleep(nap).await; + } let elapsed = start_time.elapsed(); if elapsed >= self.max_dns_wait { @@ -381,34 +958,64 @@ impl AcmeClient { "DNS propagation timeout after {elapsed:?}, max wait time is {max:?}. proceeding anyway as ACME server may have different DNS view", max = self.max_dns_wait ); + for challenge in &unsettled_challenges { + warn!("{}", challenge.unsettled_hint()); + } break; } - let dns_resolver = TokioResolver::builder_tokio() - .context("failed to create dns resolver")? - .build() - .context("failed to build dns resolver")?; - while let Some(challenge) = unsettled_challenges.pop() { - let expected_txt = &challenge.dns_value; - let settled = match dns_resolver.txt_lookup(&challenge.acme_domain).await { - Ok(record) => record.answers().iter().any(|answer| { - let RData::TXT(txt) = &answer.data else { - return false; - }; - let actual_txt = txt.to_string(); - debug!("Expected challenge: {expected_txt}, actual: {actual_txt}"); - actual_txt == *expected_txt - }), - Err(err) if err.is_no_records_found() => false, - Err(err) => { - bail!( - "failed to lookup dns record {}: {err}", + let dns_resolver = resolvers + .get(&challenge.acme_domain) + .context("no resolver for challenge domain")?; + let published = match dns_resolver.txt_lookup(&challenge.acme_domain).await { + Ok(records) => records + .answers() + .iter() + .filter_map(|answer| match &answer.data { + RData::TXT(txt) => Some(txt.to_string()), + _ => None, + }) + .collect::<Vec<_>>(), + Err(err) if err.is_no_records_found() => Vec::new(), + Err(err) if !fell_back.contains(&challenge.acme_domain) => { + // Transport failures land here rather than in the arm + // above: `is_no_records_found` covers only + // `NoRecordsFound`, so a timeout or `NoConnections` + // would otherwise abort issuance outright. The + // authoritative servers may simply be unreachable -- + // egress to :53 is often closed inside a CVM, and a + // v6-only NS set fails the same way from a v4-only host. + // Drop back to the system resolver for the rest of this + // wait: the ACME server has its own DNS view, and the + // timeout above already proceeds on expiry. + warn!( + "authoritative lookup for {} failed ({err:#}), falling back to the system resolver", challenge.acme_domain ); + fell_back.insert(challenge.acme_domain.clone()); + resolvers.insert(challenge.acme_domain.clone(), system_resolver()?); + unsettled_challenges.push(challenge); + continue 'outer; + } + Err(err) => { + // Already on the system resolver, so there is nothing + // left to fall back to. Treat it as "not settled yet" so + // the backoff below applies: re-announcing a fallback + // that already happened would both misreport the state + // and keep the delay pinned at its initial value. + debug!( + domain = &challenge.acme_domain, + "dns lookup failed: {err:#}" + ); + Vec::new() } }; - if !settled { + debug!( + "Expected challenge: {}, actual: {published:?}", + challenge.expected + ); + if !challenge.expected.satisfied_by(&published, now_secs()) { delay = Duration::from_secs(32).min(delay * 2); tries += 1; debug!( @@ -427,6 +1034,32 @@ impl AcmeClient { Ok(()) } + /// Tell the ACME server every pending challenge is answerable. + /// + /// The DNS records are published and checked first, so this is a second pass + /// over the same authorizations: 0.8 dropped `Order::set_challenge_ready(url)`, + /// and `ChallengeHandle::set_ready()` borrows the order, so the handle cannot + /// be held across the DNS wait. + async fn set_challenges_ready(&self, order: &mut Order) -> Result<()> { + let challenge_type = self.validation.challenge_type(); + let mut authorizations = order.authorizations(); + while let Some(authz) = authorizations.next().await { + let mut authz = authz.context("failed to get authorizations")?; + if authz.status != AuthorizationStatus::Pending { + continue; + } + let mut challenge = authz + .challenge(challenge_type.clone()) + .with_context(|| format!("no {challenge_type:?} challenge found"))?; + debug!("setting challenge ready for {}", challenge.url); + challenge + .set_ready() + .await + .context("failed to set challenge ready")?; + } + Ok(()) + } + async fn request_new_certificate_inner( &self, key: &str, @@ -441,9 +1074,7 @@ impl AcmeClient { .collect::<Vec<_>>(); let mut order = self .account - .new_order(&NewOrder { - identifiers: &identifiers, - }) + .new_order(&NewOrder::new(&identifiers)) .await .context("failed to cread new order")?; let mut challenges_ready = false; @@ -467,13 +1098,9 @@ impl AcmeClient { self.check_dns(challenges) .await .context("failed to check dns")?; - for challenge in &*challenges { - debug!("setting challenge ready for {}", challenge.url); - order - .set_challenge_ready(&challenge.url) - .await - .context("failed to set challenge ready")?; - } + self.set_challenges_ready(&mut order) + .await + .context("failed to set challenges ready")?; challenges_ready = true; continue; } @@ -482,7 +1109,7 @@ impl AcmeClient { debug!("order is ready, uploading CSR"); let csr = make_csr(key, domains)?; order - .finalize(csr.as_ref()) + .finalize_csr(csr.as_ref()) .await .context("failed to finalize order")?; continue; @@ -500,12 +1127,13 @@ impl AcmeClient { } // Something went wrong OrderStatus::Invalid => { - let error = find_error(&mut order).await.unwrap_or(Problem { - r#type: None, - detail: None, - status: None, - }); - bail!("order is invalid: {error}"); + let error = find_error(&mut order).await.context( + "order is invalid and its authorization error could not be retrieved", + )?; + match error { + Some(error) => bail!("order is invalid: {error}"), + None => bail!("order is invalid without error details"), + } } } } @@ -521,18 +1149,113 @@ impl AcmeClient { } } -async fn find_error(order: &mut Order) -> Option<Problem> { +/// Whether the stale TXT records under `acme_domain` still have to be cleared. +/// +/// The purge runs once per challenge name per issuance: the records this run +/// has already published live under the names in `published`, and clearing +/// those would take an answered challenge back down. +fn needs_purge(published: &[Challenge], acme_domain: &str) -> bool { + !published + .iter() + .any(|challenge| challenge.acme_domain == acme_domain) +} + +/// The name of the TXT record that answers `challenge`'s validation for `identifier`. +/// +/// The record always lives under the bare name: a wildcard authorization for +/// `*.example.com` is answered at `_acme-challenge.example.com`, and at +/// `_validation-persist.example.com` under dns-persist-01. `AuthorizedIdentifier` +/// renders the wildcard prefix in its `Display`, so formatting it directly would publish +/// the record at `_acme-challenge.*.example.com` and fail every wildcard issuance. +fn challenge_domain( + challenge: ChallengeKind, + identifier: &AuthorizedIdentifier<'_>, +) -> Result<String> { + let Identifier::Dns(name) = identifier.identifier else { + bail!("unsupported identifier type in authorization: {identifier}"); + }; + Ok(match challenge { + ChallengeKind::Dns01 => format!("_acme-challenge.{name}"), + ChallengeKind::DnsPersist01 => dns_persist::validation_domain(name), + }) +} + +/// Drop a leading underscore label, so the zone walk starts at a real name. +/// +/// Challenge records live under a reserved label -- `_acme-challenge` for +/// dns-01, `_validation-persist` for dns-persist-01 -- which never carries NS +/// records, so querying it only costs a round trip. +fn strip_challenge_label(domain: &str) -> &str { + match domain.starts_with('_') { + true => domain.split_once('.').map_or(domain, |(_, rest)| rest), + false => domain, + } +} + +async fn find_error(order: &mut Order) -> Result<Option<Problem>> { if let Some(error) = order.state().error.as_ref() { - return Some(error.clone()); + return Ok(Some(error.clone())); } - for auth in order.authorizations().await.ok()? { - for challenge in auth.challenges { - if let Some(error) = challenge.error { - return Some(error); + let mut authorizations = order.authorizations(); + while let Some(result) = authorizations.next().await { + let authz = + result.context("failed to fetch authorization while looking for the order error")?; + for challenge in &authz.challenges { + if let Some(error) = &challenge.error { + return Ok(Some(error.clone())); } } } - None + Ok(None) +} + +/// The resolver from `/etc/resolv.conf`, used to find nameservers and as the +/// fallback when the authoritative ones cannot be reached. +fn system_resolver() -> Result<TokioResolver> { + TokioResolver::builder_tokio() + .context("failed to read system dns config")? + .build() + .context("failed to build dns resolver") +} + +/// The next name up to try when a name carries no NS records. +/// +/// Stops at the last two labels. This is a heuristic, not a public-suffix +/// lookup: under a multi-label suffix such as `co.uk` it can stop on the suffix +/// itself and query the registry's nameservers, which answer with a referral +/// rather than the record. That costs one wasted lookup and then falls back, +/// which is why a PSL dependency is not worth carrying here. +fn parent_zone(name: &str) -> Option<String> { + match name.split_once('.') { + Some((_, parent)) if parent.contains('.') => Some(parent.to_string()), + _ => None, + } +} + +/// Build a resolver that queries exactly `servers`. +/// +/// Caching is disabled: this resolver exists to observe a record that was just +/// written, so a cached answer of any age is the wrong answer. +fn resolver_for(servers: &[SocketAddr]) -> Result<TokioResolver> { + let name_servers = servers + .iter() + .map(|dns_server| { + let mut name_server = NameServerConfig::udp_and_tcp(dns_server.ip()); + for connection in &mut name_server.connections { + connection.port = dns_server.port(); + } + name_server + }) + .collect(); + let mut builder = TokioResolver::builder_with_config( + ResolverConfig::from_parts(None, Vec::new(), name_servers), + TokioRuntimeProvider::default(), + ); + let options = builder.options_mut(); + options.cache_size = 0; + options.negative_min_ttl = Some(Duration::ZERO); + options.negative_max_ttl = Some(Duration::ZERO); + builder.build().context("failed to build dns resolver") } fn make_csr(key: &str, names: &[String]) -> Result<Vec<u8>> { @@ -583,6 +1306,39 @@ pub(crate) fn read_pem(cert_pem: &str) -> Result<Pem> { .context("no certificate in pem") } +/// Why the live certificate has to be reissued, or `None` if it is the one the +/// configuration asks for. +/// +/// A certificate whose names cannot be read counts as a reissue: it cannot be +/// checked against the configuration, so leaving it in place would keep serving +/// something this process can no longer reason about. +fn reissue_reason(live_cert_pem: &str, domains: &[String]) -> Option<String> { + match extract_subject_alt_names(live_cert_pem) { + Ok(names) if names_match(&names, domains) => None, + Ok(names) => Some(format!( + "the live certificate covers {}, the configuration asks for {}", + names.join(", "), + domains.join(", ") + )), + Err(err) => Some(format!("cannot read the live certificate's names: {err:#}")), + } +} + +/// Whether a certificate's DNS names are exactly the configured ones. +/// +/// Compared as sets: the order the CA returns names in is its own business, and +/// DNS names are case-insensitive and may carry a trailing root dot. Equality +/// rather than containment, so narrowing the configured list reissues too. +fn names_match(cert_names: &[String], domains: &[String]) -> bool { + fn normalized(names: &[String]) -> BTreeSet<String> { + names + .iter() + .map(|name| name.trim_end_matches('.').to_ascii_lowercase()) + .collect() + } + normalized(cert_names) == normalized(domains) +} + fn extract_subject_alt_names(cert_pem: &str) -> Result<Vec<String>> { let pem = read_pem(cert_pem)?; let cert = pem.parse_x509().context("Invalid x509 certificate")?; @@ -615,3 +1371,586 @@ fn ln_force(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> Result<()> { #[cfg(test)] mod tests; + +#[cfg(test)] +mod challenge_parsing_tests { + use instant_acme::{AuthorizationState, AuthorizationStatus, ChallengeType}; + + /// Let's Encrypt offers `dns-persist-01` alongside `dns-01`, and that + /// challenge object carries no `token`. Deserializing the authorization must + /// still succeed: `challenges` is one array, so a single unparseable entry + /// used to take the usable `dns-01` challenge down with it. + #[test] + fn an_authorization_survives_a_challenge_type_without_a_token() { + // Shape taken from a real acme-staging-v02 authorization response. + let authz = r#"{ + "identifier": { "type": "dns", "value": "example.com" }, + "status": "pending", + "expires": "2026-09-01T00:00:00Z", + "challenges": [ + { + "type": "dns-persist-01", + "url": "https://acme-staging-v02.api.letsencrypt.org/acme/chall/1/a", + "status": "pending" + }, + { + "type": "dns-01", + "url": "https://acme-staging-v02.api.letsencrypt.org/acme/chall/1/b", + "status": "pending", + "token": "a-real-token" + } + ], + "wildcard": true + }"#; + + let state: AuthorizationState = serde_json::from_str(authz) + .expect("authorization with an unknown challenge must parse"); + assert_eq!(state.status, AuthorizationStatus::Pending); + assert_eq!(state.challenges.len(), 2); + + let dns01 = state + .challenges + .iter() + .find(|c| c.r#type == ChallengeType::Dns01) + .expect("the dns-01 challenge must survive alongside the unknown one"); + assert_eq!(dns01.token, "a-real-token"); + + // The tokenless challenge parses with an empty token rather than failing. + let other = state + .challenges + .iter() + .find(|c| c.r#type != ChallengeType::Dns01) + .expect("the unknown challenge is kept"); + assert!(other.token.is_empty()); + + // A wildcard authorization still reports the bare name, which is what the + // `_acme-challenge.<name>` TXT record is published under. + let instant_acme::Identifier::Dns(name) = state.identifier().identifier else { + panic!("expected a dns identifier"); + }; + assert_eq!(name, "example.com"); + } +} + +#[cfg(test)] +mod ns_discovery_tests { + use super::parent_zone; + + #[test] + fn a_challenge_label_is_dropped_before_the_walk_starts() { + use super::strip_challenge_label; + + // Both challenge methods put their record under a reserved underscore + // label, and neither label can carry NS records. + assert_eq!( + strip_challenge_label("_acme-challenge.example.com"), + "example.com" + ); + assert_eq!( + strip_challenge_label("_validation-persist.example.com"), + "example.com" + ); + assert_eq!(strip_challenge_label("example.com"), "example.com"); + } + + #[test] + fn the_walk_climbs_to_a_name_that_can_carry_ns_records() { + // A challenge name is not a zone cut, so the walk has to climb to the + // name that actually carries the NS records. + assert_eq!(parent_zone("06rc0.kvin.wang").as_deref(), Some("kvin.wang")); + assert_eq!( + parent_zone("a.b.example.com").as_deref(), + Some("b.example.com") + ); + // The walk stops at the last two labels. That is a heuristic, not a + // public-suffix lookup: under a multi-label suffix such as `co.uk` it + // can stop on the suffix itself. In practice the registrable name + // carries NS records and the walk breaks a level earlier, and a + // referral answer just falls back, so this is a stopping rule rather + // than a correctness guarantee. + assert_eq!(parent_zone("kvin.wang"), None); + assert_eq!(parent_zone("wang"), None); + } +} + +#[cfg(test)] +mod challenge_domain_tests { + use super::{challenge_domain, ChallengeKind}; + use instant_acme::Identifier; + + /// A wildcard order authorizes the bare name with `wildcard: true`, and the TXT record + /// answering it must be published under that bare name. Formatting the identifier + /// through its `Display` instead would ask for `_acme-challenge.*.example.com`. + /// dns-persist-01 answers the same authorization under its own label, and the + /// wildcard prefix must not leak into that name either. + #[test] + fn the_challenge_domain_never_carries_a_wildcard_prefix() { + let dns = Identifier::Dns("example.com".to_string()); + for wildcard in [false, true] { + assert_eq!( + challenge_domain(ChallengeKind::Dns01, &dns.authorized(wildcard)).unwrap(), + "_acme-challenge.example.com", + "wildcard={wildcard}" + ); + assert_eq!( + challenge_domain(ChallengeKind::DnsPersist01, &dns.authorized(wildcard)).unwrap(), + "_validation-persist.example.com", + "wildcard={wildcard}" + ); + } + } + + /// Both methods are only defined for DNS identifiers; anything else is a bug in the + /// order we built, so it must be an error rather than a nonsensical record name. + #[test] + fn a_non_dns_identifier_is_rejected() { + let ip = Identifier::Ip("192.0.2.1".parse().unwrap()); + assert!(challenge_domain(ChallengeKind::Dns01, &ip.authorized(false)).is_err()); + assert!(challenge_domain(ChallengeKind::DnsPersist01, &ip.authorized(false)).is_err()); + } +} + +#[cfg(test)] +mod required_record_tests { + use super::*; + + const ACCOUNT: &str = "https://acme-v02.api.letsencrypt.org/acme/acct/1234567890"; + + fn records(challenge: ChallengeKind, domains: &[&str]) -> Vec<String> { + required_dns_records( + challenge, + dns_persist::LETS_ENCRYPT_ISSUER_DOMAIN_NAME, + ACCOUNT, + &domains.iter().map(|d| d.to_string()).collect::<Vec<_>>(), + ) + .iter() + .map(ToString::to_string) + .collect() + } + + #[test] + fn dns01_caa_content_is_byte_identical_to_the_pinned_format() { + // CAA values published by earlier releases have to keep matching, or + // issuance stops the moment this string drifts. + assert_eq!( + caa_content( + ChallengeKind::Dns01, + dns_persist::LETS_ENCRYPT_ISSUER_DOMAIN_NAME, + ACCOUNT + ), + format!("letsencrypt.org;validationmethods=dns-01;accounturi={ACCOUNT}") + ); + } + + #[test] + fn caa_content_names_the_challenge_in_use() { + // A CAA record still pinned to dns-01 refuses every dns-persist-01 order. + assert!(caa_content( + ChallengeKind::DnsPersist01, + dns_persist::LETS_ENCRYPT_ISSUER_DOMAIN_NAME, + ACCOUNT + ) + .contains("validationmethods=dns-persist-01")); + } + + #[test] + fn dns01_needs_no_validation_record() { + // certbot writes the `_acme-challenge` record itself, per order. + assert_eq!( + records(ChallengeKind::Dns01, &["*.example.com"]), + vec![ + format!( + "example.com. IN CAA 0 issue \"letsencrypt.org;validationmethods=dns-01;accounturi={ACCOUNT}\"" + ), + format!( + "example.com. IN CAA 0 issuewild \"letsencrypt.org;validationmethods=dns-01;accounturi={ACCOUNT}\"" + ), + ] + ); + } + + #[test] + fn dns_persist_asks_for_the_wildcard_policy_only_when_a_wildcard_is_ordered() { + let plain = records(ChallengeKind::DnsPersist01, &["example.com"]); + assert_eq!( + plain[0], + format!( + "_validation-persist.example.com. IN TXT \"letsencrypt.org; accounturi={ACCOUNT}\"" + ) + ); + + let wildcard = records(ChallengeKind::DnsPersist01, &["*.example.com"]); + assert_eq!( + wildcard[0], + format!( + "_validation-persist.example.com. IN TXT \"letsencrypt.org; accounturi={ACCOUNT}; policy=wildcard\"" + ) + ); + } + + #[test] + fn a_name_and_its_wildcard_share_one_validation_record() { + // Both identifiers authorize from the same base name, so asking the + // operator for two records would be asking for one too many. + let records = records( + ChallengeKind::DnsPersist01, + &["example.com", "*.example.com"], + ); + assert_eq!( + records.iter().filter(|r| r.contains(" TXT ")).count(), + 1, + "{records:#?}" + ); + assert!(records[0].contains("policy=wildcard"), "{records:#?}"); + } + + #[test] + fn each_base_name_gets_its_own_records() { + let records = records( + ChallengeKind::DnsPersist01, + &["*.a.example.com", "*.b.example.com"], + ); + assert_eq!(records.len(), 6, "{records:#?}"); + assert!(records[0].starts_with("_validation-persist.a.example.com.")); + assert!(records[3].starts_with("_validation-persist.b.example.com.")); + } +} + +#[cfg(test)] +mod purge_tests { + use super::{needs_purge, Challenge, Expected}; + + fn challenge(acme_domain: &str, dns_value: &str) -> Challenge { + Challenge { + id: Some(format!("rec-{dns_value}")), + acme_domain: acme_domain.to_string(), + expected: Expected::KeyAuthorization(dns_value.to_string()), + } + } + + /// A base name and its wildcard are two authorizations answered under one + /// `_acme-challenge.<name>`. The first one clears whatever an aborted run + /// left behind; the second must publish alongside it instead of wiping it. + #[test] + fn a_name_is_cleared_once_per_issuance() { + let mut published = vec![]; + assert!(needs_purge(&published, "_acme-challenge.example.com")); + + published.push(challenge("_acme-challenge.example.com", "value-for-base")); + assert!(!needs_purge(&published, "_acme-challenge.example.com")); + } + + /// A SAN list can span zones, and clearing one challenge name says nothing + /// about the others. + #[test] + fn an_untouched_name_is_still_cleared() { + let published = vec![challenge("_acme-challenge.example.com", "value-for-base")]; + assert!(needs_purge(&published, "_acme-challenge.example.org")); + } +} + +/// The owned name list the functions under test take, spelled once. +#[cfg(test)] +fn names(list: &[&str]) -> Vec<String> { + list.iter().map(|name| name.to_string()).collect() +} + +#[cfg(test)] +mod names_match_tests { + use super::{names, names_match}; + + /// The CA returns the names in whatever order it likes -- the staging CA + /// puts the wildcard first -- and DNS names are case-insensitive and may + /// carry the root dot. None of that is a configuration change. + #[test] + fn the_same_names_written_differently_are_the_same_names() { + assert!(names_match( + &names(&["*.example.com", "Example.com."]), + &names(&["example.com", "*.example.com"]), + )); + } + + /// Adding a name to the configuration is what has to trigger a reissue -- + /// this is the case that silently did nothing before. + #[test] + fn an_added_name_does_not_match() { + assert!(!names_match( + &names(&["example.com"]), + &names(&["example.com", "*.example.com"]), + )); + } + + /// Dropping a name must reissue as well: a certificate covering more than + /// the configuration asks for is not the certificate that was asked for. + #[test] + fn a_dropped_name_does_not_match() { + assert!(!names_match( + &names(&["example.com", "*.example.com"]), + &names(&["example.com"]), + )); + } +} + +#[cfg(test)] +mod reissue_reason_tests { + use super::{names, reissue_reason}; + use rcgen::{CertificateParams, KeyPair}; + + /// A certificate carrying exactly `sans`, in the order given -- the CA's own + /// order is not the configuration's, which is what the decision has to + /// tolerate. + fn cert_with_names(sans: &[&str]) -> String { + let key = KeyPair::generate().expect("failed to generate key"); + CertificateParams::new(names(sans)) + .expect("failed to build certificate params") + .self_signed(&key) + .expect("failed to self-sign") + .pem() + } + + #[test] + fn the_configured_certificate_is_kept() { + let cert = cert_with_names(&["*.example.com", "example.com"]); + assert_eq!( + reissue_reason(&cert, &names(&["example.com", "*.example.com"])), + None + ); + } + + /// The case that silently did nothing before: a name added to the + /// configuration has to reach the CA. + #[test] + fn a_certificate_missing_a_configured_name_is_reissued() { + let cert = cert_with_names(&["example.com"]); + let reason = reissue_reason(&cert, &names(&["example.com", "*.example.com"])) + .expect("an added name must reissue"); + // Both lists are in the message: the operator has to be able to see + // what is being replaced and why. + assert!(reason.contains("covers example.com"), "{reason}"); + assert!( + reason.contains("asks for example.com, *.example.com"), + "{reason}" + ); + } + + /// A certificate whose names cannot be read cannot be checked against the + /// configuration either, so it is replaced rather than served on. + #[test] + fn an_unreadable_certificate_is_reissued() { + let reason = reissue_reason("not a certificate", &names(&["example.com"])) + .expect("an unreadable certificate must reissue"); + assert!(reason.contains("cannot read"), "{reason}"); + } +} + +#[cfg(test)] +mod caa_guard_tests { + use super::{caa_tag, is_caa_guard}; + + /// The guard is what an interrupted `set_caa_records` leaves behind, and a + /// rerun has to recognize its own leftovers to be able to replace them. + #[test] + fn a_guard_is_recognized_however_the_provider_renders_it() { + assert!(is_caa_guard(r#"0 issue ";""#)); + assert!(is_caa_guard(r#"0 issuewild ";""#)); + assert!(is_caa_guard("0 issue ;")); + assert!(is_caa_guard("128 issue \";\"")); + } + + /// A real issuer record must never be mistaken for a guard: sweeping one + /// early would delete the zone's only valid CAA before its replacement is + /// written. + #[test] + fn a_real_issuer_record_is_not_a_guard() { + assert!(!is_caa_guard( + r#"0 issue "letsencrypt.org;validationmethods=dns-01;accounturi=https://acme.example/acct/1""# + )); + assert!(!is_caa_guard(r#"0 issue "letsencrypt.org""#)); + // `iodef` is neither an issuer constraint nor a guard. + assert!(!is_caa_guard(r#"0 iodef "mailto:x@example.com""#)); + // A value that merely starts with the guard's character. + assert!(!is_caa_guard(r#"0 issue ";extra""#)); + // Trailing junk means it is not the record this wrote. + assert!(!is_caa_guard(r#"0 issue ";" extra"#)); + assert!(!is_caa_guard("")); + } + + /// The sweep is scoped by the same tag test the replacement pass uses. + #[test] + fn the_guard_tags_are_the_ones_the_replacement_pass_replaces() { + for content in [r#"0 issue ";""#, r#"0 issuewild ";""#] { + let tag = caa_tag(content).expect("a rendered CAA record has a tag"); + assert!(matches!(tag, "issue" | "issuewild"), "{tag}"); + } + } +} + +/// The DNS wait budget: what `max_dns_wait` is clamped to, and what the poll +/// loop does with the result. +/// +/// Both halves are pure and both were uncovered. `advisory_dns_wait` exists +/// only for the clamp, and nothing asserted it; `dns_poll_sleep` is the rule the +/// loop's own comment describes and could not be reached without a resolver and +/// a live challenge. +#[cfg(test)] +mod dns_wait_tests { + use super::{advisory_dns_wait, dns_poll_sleep, DNS_WAIT_SHARE_OF_RENEW_TIMEOUT}; + use std::time::Duration; + + const fn secs(n: u64) -> Duration { + Duration::from_secs(n) + } + + /// A wait already inside the budget is left where the operator put it: the + /// clamp is a ceiling, not an override. + #[test] + fn a_wait_that_already_fits_is_unchanged() { + assert_eq!(advisory_dns_wait(secs(30), secs(600)), secs(30)); + assert_eq!(advisory_dns_wait(secs(5), secs(300)), secs(5)); + } + + /// The CLI's own defaults are the failing case: a 300s wait inside a 120s + /// renewal budget never reaches its "proceed anyway" exit, so an unanswered + /// check ends as a timeout rather than as the warning naming the record. + #[test] + fn the_wait_ends_before_the_renewal_budget_does() { + let renew = secs(120); + let wait = advisory_dns_wait(secs(300), renew); + assert_eq!(wait, secs(60)); + assert!(wait < renew, "{wait:?}"); + } + + /// The boundary belongs to the configured value: exactly half the renewal + /// budget still fits. + #[test] + fn a_wait_of_exactly_half_the_renewal_budget_fits() { + assert_eq!(advisory_dns_wait(secs(150), secs(300)), secs(150)); + } + + /// Over the ceiling, the ceiling wins. + #[test] + fn a_wait_that_does_not_fit_is_cut_to_the_ceiling() { + assert_eq!(advisory_dns_wait(secs(300), secs(60)), secs(30)); + assert_eq!(advisory_dns_wait(secs(3600), secs(300)), secs(150)); + } + + /// The case the clamp was written for: the stock defaults on both sides are + /// 300s, so without it the wait and the timeout wrapping the order expire + /// together and the outer one always wins -- the renewal dies as a bare + /// timeout and the warning naming the missing record is never logged. + /// + /// What matters is not the number but that the wait ends strictly first. + #[test] + fn the_stock_defaults_leave_room_for_the_graceful_exit() { + let renew = secs(300); + let wait = advisory_dns_wait(secs(300), renew); + assert_eq!(wait, secs(150)); + assert!( + wait < renew, + "the DNS wait must end before the timeout wrapping the order" + ); + } + + /// Zero configured means the check is off, and the clamp must not quietly + /// turn it back on. + /// + /// Reachable from the certbot CLI, whose `max_dns_wait` is a plain + /// `#[serde(default)]` field with no lower bound -- unlike the gateway's + /// `CreateDnsCredential`, which refuses zero with "max_dns_wait must be + /// greater than zero". So the clamp is the only thing standing between a + /// zero in a config file and whatever the poll loop would do with it, and + /// what it must do is pass the zero through: the loop's own reading of a + /// spent budget is what turns the check off, and a clamp that rounded zero + /// up would re-enable a check the operator switched off. + #[test] + fn zero_stays_zero() { + assert_eq!(advisory_dns_wait(secs(0), secs(300)), secs(0)); + } + + /// A renewal budget of zero leaves nothing for the wait, and must not + /// underflow or panic on the division. + #[test] + fn a_zero_renewal_budget_leaves_no_wait() { + assert_eq!(advisory_dns_wait(secs(300), secs(0)), secs(0)); + assert_eq!(advisory_dns_wait(secs(0), secs(0)), secs(0)); + } + + /// The two invariants, over the whole grid rather than at the points above: + /// never more than what was configured, and never more than the share of the + /// renewal budget the wait is allowed. + #[test] + fn the_result_never_exceeds_either_bound() { + for configured in [0, 1, 5, 59, 60, 61, 150, 299, 300, 3600] { + for renew in [0, 1, 2, 60, 119, 120, 300, 301] { + let (configured, renew) = (secs(configured), secs(renew)); + let got = advisory_dns_wait(configured, renew); + assert!(got <= configured, "{got:?} > configured {configured:?}"); + assert!( + got <= renew / DNS_WAIT_SHARE_OF_RENEW_TIMEOUT, + "{got:?} exceeds its share of {renew:?}" + ); + } + } + } + + /// With budget to spare, the poll sleeps the full backoff step. + #[test] + fn a_poll_with_budget_to_spare_sleeps_the_whole_backoff() { + assert_eq!( + dns_poll_sleep(secs(150), secs(10), Duration::from_millis(250)), + Duration::from_millis(250) + ); + assert_eq!(dns_poll_sleep(secs(150), secs(10), secs(32)), secs(32)); + } + + /// Near the end of the budget the sleep is cut to what is left. + /// + /// The regression this guards: sleeping the full step and checking after + /// overshoots by up to 32s, which is enough to hand the deadline to the + /// timeout wrapping the order. + #[test] + fn a_poll_near_the_deadline_sleeps_only_what_is_left() { + assert_eq!(dns_poll_sleep(secs(60), secs(58), secs(32)), secs(2)); + assert_eq!( + dns_poll_sleep(secs(5), secs(4), secs(32)), + secs(1), + "the e2e suite's 5s budget must not be overrun by a 32s backoff step" + ); + } + + /// A spent budget sleeps not at all, so the caller reaches its expiry check + /// on the same pass. + #[test] + fn a_spent_budget_sleeps_not_at_all() { + assert_eq!(dns_poll_sleep(secs(60), secs(60), secs(32)), secs(0)); + assert_eq!(dns_poll_sleep(secs(60), secs(90), secs(32)), secs(0)); + } + + /// `max_dns_wait = 0` skips the check rather than running it once: the first + /// pass sleeps nothing and the caller's `elapsed >= budget` is already true, + /// so no lookup is ever made. + #[test] + fn a_zero_budget_skips_the_check_entirely() { + let budget = secs(0); + assert_eq!(dns_poll_sleep(budget, secs(0), secs(32)), secs(0)); + assert!( + secs(0) >= budget, + "the caller's expiry check fires on the first pass" + ); + } + + /// The sleep never runs past the budget, for any point inside it. + #[test] + fn a_poll_never_sleeps_past_the_budget() { + for budget in [0u64, 1, 5, 60, 150] { + for elapsed in 0..=budget + 2 { + for backoff in [1u64, 2, 4, 8, 16, 32] { + let (b, e) = (secs(budget), secs(elapsed)); + let nap = dns_poll_sleep(b, e, secs(backoff)); + assert!(e + nap <= b.max(e), "{e:?} + {nap:?} overruns {b:?}"); + assert!(nap <= secs(backoff)); + } + } + } + } +} diff --git a/dstack/certbot/src/bot.rs b/dstack/certbot/src/bot.rs index 76cd8491a..2ead321b6 100644 --- a/dstack/certbot/src/bot.rs +++ b/dstack/certbot/src/bot.rs @@ -9,14 +9,16 @@ use std::{ time::Duration, }; -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use fs_err as fs; use tokio::time::sleep; -use tracing::{error, info}; +use tracing::{error, info, warn}; -use crate::acme_client::{acme_matches, read_pem}; +use crate::acme_client::{acme_matches, read_pem, ChallengeKind, RequiredRecord, ValidationMethod}; +use crate::dns_persist::{resolve_issuer_domain_name, LETS_ENCRYPT_ISSUER_DOMAIN_NAME}; use super::{AcmeClient, Dns01Client}; +use crate::acme_client::advisory_dns_wait; #[allow(clippy::duplicated_attributes)] #[derive(Clone, Debug, bon::Builder)] @@ -27,6 +29,15 @@ pub struct CertBotConfig { auto_set_caa: bool, credentials_file: PathBuf, auto_create_account: bool, + /// ACME challenge used to prove control of the domains. + #[builder(default)] + challenge: ChallengeKind, + /// Issuer Domain Name naming the CA in `dns-persist-01` and CAA records. + /// + /// Must be one of the `issuer-domain-names` the CA sends in the challenge. + #[builder(default = LETS_ENCRYPT_ISSUER_DOMAIN_NAME.to_string())] + issuer_domain_name: String, + /// Cloudflare API token. Unused, and warned about, under `dns-persist-01`. cf_api_token: String, cf_api_url: Option<String>, cert_file: PathBuf, @@ -57,17 +68,12 @@ pub struct CertBot { async fn create_new_account( config: &CertBotConfig, - dns01_client: Dns01Client, + validation: ValidationMethod, ) -> Result<AcmeClient> { info!("creating new ACME account"); - let client = AcmeClient::new_account( - &config.acme_url, - dns01_client, - config.max_dns_wait, - config.dns_txt_ttl, - ) - .await - .context("failed to create new account")?; + let client = AcmeClient::new_account(&config.acme_url, validation, dns_wait(config)) + .await + .context("failed to create new account")?; let credentials = client .dump_credentials() .context("failed to dump credentials")?; @@ -87,39 +93,20 @@ async fn create_new_account( impl CertBot { /// Build a new `CertBot` from a `CertBotConfig`. pub async fn build(config: CertBotConfig) -> Result<Self> { - let base_domain = config - .cert_subject_alt_names - .first() - .context("cert_subject_alt_names is empty")? - .trim() - .trim_start_matches("*.") - .trim_end_matches('.') - .to_string(); - let dns01_client = Dns01Client::new_cloudflare( - base_domain, - config.cf_api_token.clone(), - config.cf_api_url.clone(), - ) - .await?; + let validation = build_validation_method(&config).await?; let acme_client = match fs::read_to_string(&config.credentials_file) { Ok(credentials) => { if acme_matches(&credentials, &config.acme_url) { - AcmeClient::load( - dns01_client, - &credentials, - config.max_dns_wait, - config.dns_txt_ttl, - ) - .await? + AcmeClient::load(validation, &credentials, dns_wait(&config)).await? } else { - create_new_account(&config, dns01_client).await? + create_new_account(&config, validation).await? } } Err(e) if e.kind() == ErrorKind::NotFound => { if !config.auto_create_account { return Err(e).context("credentials file not found"); } - create_new_account(&config, dns01_client).await? + create_new_account(&config, validation).await? } Err(e) => { return Err(e).context("failed to read credentials file"); @@ -149,37 +136,35 @@ impl CertBot { /// Run the certbot. pub async fn run(&self) { loop { - match self.renew(false).await { - Ok(renewed) => { - if !renewed { - continue; - } - if let Some(hook) = &self.config.renewed_hook { - info!("running renewed hook"); - let result = std::process::Command::new("/bin/sh") - .arg("-c") - .arg(hook) - .status(); - match result { - Ok(status) => { - if !status.success() { - error!("renewed hook failed with status: {status}"); - } - } - Err(err) => { - error!("failed to run renewed hook: {err:?}"); - } - } - } - } - Err(e) => { - error!("failed to run certbot: {e:?}"); - } + if let Err(error) = self.renew_and_run_hook(false).await { + error!("failed to run certbot: {error:?}"); } sleep(self.config.renew_interval).await; } } + /// Run one renewal attempt and invoke the configured hook after a commit. + pub async fn renew_and_run_hook(&self, force: bool) -> Result<bool> { + let renewed = self.renew(force).await?; + if !renewed { + return Ok(false); + } + let Some(hook) = &self.config.renewed_hook else { + return Ok(true); + }; + info!("running renewed hook"); + match std::process::Command::new("/bin/sh") + .arg("-c") + .arg(hook) + .status() + { + Ok(status) if status.success() => {} + Ok(status) => error!("renewed hook failed with status: {status}"), + Err(error) => error!("failed to run renewed hook: {error:?}"), + } + Ok(true) + } + /// Run the certbot once. pub async fn renew(&self, force: bool) -> Result<bool> { tokio::time::timeout(self.config.renew_timeout, self.renew_inner(force)) @@ -192,7 +177,8 @@ impl CertBot { } async fn renew_inner(&self, force: bool) -> Result<bool> { - let created = self + let live_cert_exists = self.config.cert_file.exists() && self.config.key_file.exists(); + let issued = self .acme_client .create_cert_if_needed( &self.config.cert_subject_alt_names, @@ -200,11 +186,30 @@ impl CertBot { &self.config.key_file, &self.config.cert_dir, ) - .await?; - if created { - info!("created new certificate"); - return Ok(true); - } + .await; + // A live certificate that does not cover the configured names is + // reissued above, and that reissuance keeps failing for as long as the + // configuration names something the CA will not validate -- a typo, a + // zone the DNS credentials cannot write. Failing the run right here + // would take the renewal check below down with it, so one name the + // operator got wrong would stop renewing the certificate that is + // actually being served, until it expires. The renewal still runs; the + // error is reported, and returned below unless the renewal committed + // something of its own. + let reissue_error = match issued { + Ok(true) => { + info!("created new certificate"); + return Ok(true); + } + Ok(false) => None, + // Nothing is being served yet, so there is no renewal to protect + // and `auto_renew` has no certificate to read. + Err(err) if !live_cert_exists => return Err(err), + Err(err) => { + error!("failed to issue a certificate for the configured domains: {err:#}"); + Some(err) + } + }; info!("checking if certificate needs to be renewed"); let renewed = self .acme_client @@ -217,21 +222,25 @@ impl CertBot { ) .await?; - match renewed { - true => { + match (renewed, reissue_error) { + (true, _) => { info!( "renewed certificate for {}", self.config.cert_file.display() ); + Ok(true) } - false => { + // The renewal committed nothing, so the reissue failure is the + // whole outcome of this run and `renew --once` must report it. + (false, Some(err)) => Err(err), + (false, None) => { info!( "certificate {} is up to date", self.config.cert_file.display() ); + Ok(false) } } - Ok(renewed) } /// Set CAA record for the domain. @@ -240,6 +249,96 @@ impl CertBot { .set_caa_records(&self.config.cert_subject_alt_names) .await } + + /// The DNS records that have to exist for the configured domains. + pub fn required_dns_records(&self) -> Vec<RequiredRecord> { + self.acme_client + .required_dns_records(&self.config.cert_subject_alt_names) + } +} + +/// The DNS wait this configuration should actually use. +/// +/// `renew_timeout` wraps the whole renewal here exactly as it does in the +/// gateway, and the defaults are skewed the same way -- further, in fact: +/// `max_dns_wait` defaults to 300s against a 120s renewal budget, so an +/// unanswered check runs the renewal into its timeout every time instead of +/// reporting the record it could not see. +fn dns_wait(config: &CertBotConfig) -> Duration { + advisory_dns_wait(config.max_dns_wait, config.renew_timeout) +} + +/// The Issuer Domain Name this configuration names, checked before it is used. +/// +/// Both challenges read the same setting, so both get the same treatment: empty +/// means the default, and a value that would not survive being written into a +/// CAA or validation record is refused here rather than at the point it would +/// corrupt a zone. +fn issuer_domain_name(config: &CertBotConfig) -> Result<String> { + resolve_issuer_domain_name(&config.issuer_domain_name) + .context("invalid issuer_domain_name in the certbot configuration") +} + +/// Resolve the configured challenge into a live validation method. +/// +/// `dns-01` resolves the Cloudflare zone here, which is an authenticated call, +/// so a bad credential fails at startup rather than at the first renewal. +/// `dns-persist-01` talks to no provider at all. +async fn build_validation_method(config: &CertBotConfig) -> Result<ValidationMethod> { + match config.challenge { + ChallengeKind::Dns01 => { + // Named here rather than left to the provider. `cf_api_token` is + // `#[serde(default)]` so a dns-persist-01 config can omit it, which + // also means a dns-01 config that forgets it no longer fails + // deserialization -- it reaches Cloudflare and comes back as an + // "Invalid format for Authorization header", naming the header + // instead of the setting the operator has to add. + if config.cf_api_token.is_empty() { + bail!( + "cf_api_token is required with dns-01, which proves control by writing a \ + TXT record through the DNS provider; set it, or switch to \ + challenge = \"dns-persist-01\", which needs no provider credential" + ); + } + let base_domain = config + .cert_subject_alt_names + .first() + .context("cert_subject_alt_names is empty")? + .trim() + .trim_start_matches("*.") + .trim_end_matches('.') + .to_string(); + let client = Dns01Client::new_cloudflare( + base_domain, + config.cf_api_token.clone(), + config.cf_api_url.clone(), + ) + .await?; + Ok(ValidationMethod::Dns01 { + client, + txt_ttl: config.dns_txt_ttl, + issuer_domain_name: issuer_domain_name(config)?, + }) + } + ChallengeKind::DnsPersist01 => { + // Refuse rather than silently skip: `auto_set_caa` promises the CAA + // records are kept in sync, and without DNS write access nothing here + // can keep that promise. `certbot dns-records` prints what to publish. + if config.auto_set_caa { + bail!( + "auto_set_caa is not supported with dns-persist-01, which has no DNS \ + write access; set auto_set_caa = false and publish the records from \ + `certbot dns-records` by hand" + ); + } + if !config.cf_api_token.is_empty() { + warn!("ignoring cf_api_token: dns-persist-01 needs no DNS provider credential"); + } + Ok(ValidationMethod::DnsPersist01 { + issuer_domain_name: issuer_domain_name(config)?, + }) + } + } } pub fn read_pubkey(cert_pem: &str) -> Result<Vec<u8>> { @@ -259,6 +358,7 @@ pub fn list_certs(workdir: impl AsRef<Path>) -> Result<Vec<PathBuf>> { certs.push(cert_path); } } + certs.sort(); Ok(certs) } @@ -274,3 +374,37 @@ pub fn list_cert_public_keys(workdir: impl AsRef<Path>) -> Result<BTreeSet<Vec<u #[cfg(test)] mod tests; + +#[cfg(test)] +mod listing_tests { + use super::list_certs; + use fs_err as fs; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn certificate_directories_are_listed_in_stable_order() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "dstack-certbot-list-{}-{nonce}", + std::process::id() + )); + for name in ["0002", "0001"] { + let directory = root.join(name); + fs::create_dir_all(&directory).unwrap(); + fs::write(directory.join("cert.pem"), name).unwrap(); + } + + let listed = list_certs(&root).unwrap(); + + assert_eq!( + listed, + ["0001", "0002"] + .map(|name| root.join(name).join("cert.pem")) + .to_vec() + ); + fs::remove_dir_all(root).unwrap(); + } +} diff --git a/dstack/certbot/src/dns_persist.rs b/dstack/certbot/src/dns_persist.rs new file mode 100644 index 000000000..ec21b0e98 --- /dev/null +++ b/dstack/certbot/src/dns_persist.rs @@ -0,0 +1,617 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! The persistent DNS authorization record used by the `dns-persist-01` challenge. +//! +//! `dns-persist-01` (draft-ietf-acme-dns-persist-01) replaces the per-order +//! `_acme-challenge` TXT record of `dns-01` with a single record that stays in the +//! zone: `_validation-persist.<name>`, naming the CA and the ACME account allowed +//! to issue for that name. The account key proves who is asking; the record proves +//! the zone owner agreed. Nothing about it changes between orders, so a client that +//! uses this method never needs write access to the zone. +//! +//! This module owns the record's syntax: rendering the line an operator has to +//! publish, and deciding whether what is currently published would satisfy the CA. +//! The RDATA is an RFC 8659 `issue-value` — the same grammar as a CAA `issue` +//! record — so the parser here mirrors the one CAs run (see `va/dns_persist.go` in +//! Boulder), including the parts that reject rather than ignore: a trailing +//! semicolon, a repeated tag, whitespace inside a value. + +use std::fmt; + +use anyhow::{bail, Context, Result}; + +/// Label prepended to the name being validated to form the validation domain name. +/// +/// draft-ietf-acme-dns-persist-01, section 4. +const VALIDATION_LABEL: &str = "_validation-persist"; + +/// Issuer Domain Name for Let's Encrypt, matching the `caaIdentities` it advertises. +/// +/// A CA lists the names it answers to in the challenge object's +/// `issuer-domain-names`; a record naming anything else is ignored by that CA. +pub const LETS_ENCRYPT_ISSUER_DOMAIN_NAME: &str = "letsencrypt.org"; + +/// The Issuer Domain Name to use, given what an operator configured. +/// +/// Empty means the default, which is what an untouched configuration carries. +/// Anything else is checked before it is believed: the name is interpolated +/// straight into an RFC 8659 `issue-value`, both in a `_validation-persist` +/// record and in the CAA records certbot publishes, and `set_caa_records` +/// installs the new value *after* deleting the records it replaces. A value +/// carrying a `;`, a space, or a `#` does not merely fail to authorize -- it +/// leaves the zone holding a malformed `issue` property with no valid record +/// behind it, which is CAA that forbids every issuer. Rejecting it at the point +/// it is configured keeps that out of the zone entirely. +pub fn resolve_issuer_domain_name(configured: &str) -> Result<String> { + let name = configured.trim(); + if name.is_empty() { + return Ok(LETS_ENCRYPT_ISSUER_DOMAIN_NAME.to_string()); + } + let name = name.trim_end_matches('.'); + if name.is_empty() || name.len() > 253 { + bail!("issuer domain name must be between 1 and 253 characters: {configured:?}"); + } + for label in name.split('.') { + // RFC 8659 spells a label `(ALPHA / DIGIT) *( *("-") (ALPHA / DIGIT) )`: + // alphanumeric at both ends, hyphens only between them, and nothing + // else -- no underscore, which is a DNS convention for names that are + // never issuer names. A CA holding the record to that grammar reads + // anything looser as a malformed `issue` property, which is the state + // this check exists to keep out of a zone. + // Each rejection names what it rejected: the operator has to fix the + // value, and "malformed" alone does not say which character to remove -- + // an underscore, say, which is legal in plenty of DNS names and is the + // one most likely to be reached for here. + let bytes = label.as_bytes(); + let context = "it is written verbatim into CAA and validation records, \ + where a malformed label makes the whole record unparseable"; + if bytes.is_empty() { + bail!("issuer domain name {configured:?} has an empty label: {context}"); + } + if bytes.len() > 63 { + bail!("issuer domain name {configured:?} has a label over 63 octets: {context}"); + } + // Scanned as characters, not bytes: a byte cast to `char` renders a + // multi-byte character's first byte as some unrelated Latin-1 one, so + // `lé.org` would be rejected for a `Ã` that appears nowhere in the + // value the same message quotes back. + if let Some(ch) = label + .chars() + .find(|ch| !ch.is_ascii_alphanumeric() && *ch != '-') + { + bail!( + "issuer domain name {configured:?} contains {ch:?}, which a DNS label cannot: \ + {context}" + ); + } + if !bytes[0].is_ascii_alphanumeric() || !bytes[bytes.len() - 1].is_ascii_alphanumeric() { + bail!( + "issuer domain name {configured:?} has a label starting or ending with a \ + hyphen: {context}" + ); + } + } + Ok(name.to_string()) +} + +/// The `policy` value that widens a record to cover wildcards. +const POLICY_WILDCARD: &str = "wildcard"; + +/// The validation domain name for `name`, where the CA looks for the TXT record. +/// +/// A wildcard request is authorized by the record on its base name: ACME strips the +/// `*.` before creating the authorization, so `*.example.com` and `example.com` +/// share `_validation-persist.example.com` and are told apart by `policy=wildcard`. +pub fn validation_domain(name: &str) -> String { + let base = name.strip_prefix("*.").unwrap_or(name); + format!("{VALIDATION_LABEL}.{base}") +} + +/// The record an operator has to publish for one name. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AuthorizationRecord { + /// Issuer Domain Name of the CA this record authorizes. + pub issuer_domain_name: String, + /// URI of the ACME account allowed to issue, compared byte for byte by the CA. + pub account_uri: String, + /// Whether the record also covers `*.<name>`. + pub wildcard: bool, +} + +impl AuthorizationRecord { + /// The TXT RDATA to publish, as a single string. + /// + /// Rendered without a trailing semicolon: CAs read the RDATA as an RFC 8659 + /// `issue-value`, where a trailing semicolon is an empty parameter and makes + /// the whole record malformed. + pub fn rdata(&self) -> String { + let mut rdata = format!( + "{}; accounturi={}", + self.issuer_domain_name, self.account_uri + ); + if self.wildcard { + rdata.push_str("; policy=wildcard"); + } + rdata + } + + /// Whether `rdata` currently published at the validation domain satisfies this + /// record's requirement, as of `now` (UNIX seconds). + /// + /// Follows the CA's own filter: a record naming a different issuer is not a + /// failure, it belongs to another CA and is skipped. Only a record that names + /// our issuer is held to the account, policy and lifetime checks. + fn satisfied_by(&self, rdata: &str, now: u64) -> bool { + let Ok(parsed) = IssueValue::parse(rdata) else { + return false; + }; + if normalized_issuer(&parsed.issuer_domain_name) + != normalized_issuer(&self.issuer_domain_name) + { + return false; + } + if parsed.account_uri != self.account_uri { + return false; + } + if parsed.persist_until.is_some_and(|until| now > until) { + return false; + } + // A record without `policy=wildcard` authorizes the exact name only, so it + // cannot stand in for a wildcard request. The reverse is fine: a wildcard + // record also covers the name itself. + !self.wildcard || parsed.policy.as_deref().is_some_and(is_wildcard_policy) + } + + /// Whether any of the TXT records at the validation domain satisfies this one. + /// + /// Several records may sit at the same label, one per CA or per account, and + /// the CA accepts the name if any single record passes. + pub fn satisfied_by_any(&self, published: &[String], now: u64) -> bool { + published.iter().any(|rdata| self.satisfied_by(rdata, now)) + } +} + +impl fmt::Display for AuthorizationRecord { + /// Renders the full zone-file line, ready to paste into a DNS provider. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "IN TXT \"{}\"", self.rdata()) + } +} + +/// A parsed RFC 8659 `issue-value`, the grammar shared with CAA `issue` records. +#[derive(Debug, PartialEq, Eq)] +struct IssueValue { + issuer_domain_name: String, + account_uri: String, + policy: Option<String>, + /// UNIX timestamp after which the CA stops accepting the record. + persist_until: Option<u64>, +} + +impl IssueValue { + fn parse(rdata: &str) -> Result<Self> { + let mut parts = rdata.split(';'); + let issuer_domain_name = trim_wsp(parts.next().unwrap_or_default()); + if issuer_domain_name.is_empty() { + bail!("missing issuer domain name"); + } + + let mut account_uri = None; + let mut policy = None; + let mut persist_until = None; + let mut seen = Vec::new(); + for part in parts { + let part = trim_wsp(part); + // An empty parameter means a doubled or trailing semicolon. CAs treat + // that as malformed rather than skipping it, so neither do we. + if part.is_empty() { + bail!("empty parameter or trailing semicolon"); + } + let (tag, value) = part.split_once('=').context("parameter is not tag=value")?; + // RFC 8659's grammar is `parameter = tag *WSP "=" *WSP value`, so the + // separator may be padded on either side and each half is trimmed on + // its own. Trimming only the whole parameter would read the tag of + // `accounturi = <uri>` as `"accounturi "` and report the mandatory + // parameter missing on a record the CA accepts. + let tag = trim_wsp(tag).to_lowercase(); + let value = trim_wsp(value); + if tag.is_empty() { + bail!("parameter has an empty tag"); + } + if !value.bytes().all(is_value_byte) { + bail!("parameter {tag} has a value with a forbidden character"); + } + match tag.as_str() { + "accounturi" | "policy" | "persistuntil" => { + // Only recognized tags are held to uniqueness: the draft has + // the CA ignore unknown tags outright, so a record repeating + // one is still a record the CA issues from. + if seen.contains(&tag) { + bail!("duplicate parameter {tag}"); + } + seen.push(tag.clone()); + } + // The draft requires unrecognized tags to be ignored, so that + // later revisions can add parameters without invalidating records. + _ => continue, + } + match tag.as_str() { + "accounturi" => { + if value.is_empty() { + bail!("empty value for the mandatory accounturi parameter"); + } + account_uri = Some(value.to_string()); + } + "policy" => policy = Some(value.to_string()), + "persistuntil" => { + persist_until = Some( + value + .parse::<u64>() + .context("persistUntil is not a base-10 timestamp")?, + ) + } + _ => {} + } + } + + Ok(Self { + issuer_domain_name: issuer_domain_name.to_string(), + account_uri: account_uri.context("missing mandatory accounturi parameter")?, + policy, + persist_until, + }) + } +} + +/// An Issuer Domain Name folded the way the CA folds it before comparing. +/// +/// Boulder normalizes both sides (lowercase, then drop the root dot) before +/// deciding whether a record is one of its own, so `LetsEncrypt.ORG.` names the +/// same CA as `letsencrypt.org`. Comparing raw bytes here would treat a record +/// the CA honours as belonging to someone else and warn about a missing record +/// through every issuance. IDNA folding is left out: this compares against a +/// name dstack configures, and an operator writing a non-ASCII issuer name has +/// a mismatch a self-check cannot paper over. +fn normalized_issuer(name: &str) -> String { + name.trim_end_matches('.').to_lowercase() +} + +fn is_wildcard_policy(policy: &str) -> bool { + policy.eq_ignore_ascii_case(POLICY_WILDCARD) +} + +/// Trim the whitespace RFC 8659 allows around the issuer name and each parameter. +fn trim_wsp(part: &str) -> &str { + part.trim_matches([' ', '\t']) +} + +/// Whether a byte may appear in a parameter value. +/// +/// RFC 8659 allows printable ASCII except `;`, which excludes whitespace: a value +/// containing a space is a malformed record, not a value with a space in it. +fn is_value_byte(byte: u8) -> bool { + matches!(byte, 0x21..=0x3a | 0x3c..=0x7e) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// An arbitrary "now" that precedes every `persistUntil` used below. + const NOW: u64 = 1_700_000_000; + const ACCOUNT: &str = "https://acme-v02.api.letsencrypt.org/acme/acct/1234567890"; + + fn record(wildcard: bool) -> AuthorizationRecord { + AuthorizationRecord { + issuer_domain_name: LETS_ENCRYPT_ISSUER_DOMAIN_NAME.to_string(), + account_uri: ACCOUNT.to_string(), + wildcard, + } + } + + /// Whether `published` satisfies a request for `name`, wildcard or not. + fn accepts(wildcard: bool, published: &str) -> bool { + record(wildcard).satisfied_by(published, NOW) + } + + /// RFC 8659 spells the parameter grammar `tag *WSP "=" *WSP value`, and + /// Boulder trims each half separately. A record padded around the separator + /// is one the CA issues from, so the self-check has to read it the same way. + #[test] + fn whitespace_around_the_separator_is_not_part_of_the_tag_or_value() { + assert!(accepts( + false, + &format!("letsencrypt.org; accounturi = {ACCOUNT}") + )); + assert!(accepts( + true, + &format!("letsencrypt.org;\taccounturi\t=\t{ACCOUNT}; policy\t=\twildcard") + )); + } + + /// The draft has the CA ignore unrecognized tags outright, so repeating one + /// cannot invalidate a record. Uniqueness is only enforced where the CA + /// enforces it. + #[test] + fn a_repeated_unknown_tag_is_ignored_rather_than_fatal() { + assert!(accepts( + false, + &format!("letsencrypt.org; accounturi={ACCOUNT}; futuretag=a; futuretag=b") + )); + assert!(!accepts( + false, + &format!("letsencrypt.org; accounturi={ACCOUNT}; accounturi={ACCOUNT}") + )); + } + + /// Boulder folds case and drops the root dot before deciding whether a + /// record names it, so the same record must not read as another CA's here. + #[test] + fn the_issuer_name_is_compared_the_way_the_ca_compares_it() { + assert!(accepts( + false, + &format!("LetsEncrypt.ORG.; accounturi={ACCOUNT}") + )); + assert!(!accepts( + false, + &format!("other-ca.example; accounturi={ACCOUNT}") + )); + } + + /// `accounturi` is mandatory, and an empty value does not supply it. + #[test] + fn an_empty_account_uri_is_not_an_account_uri() { + assert!(!accepts(false, "letsencrypt.org; accounturi=")); + } + + /// A label longer than DNS allows is refused, and one right at the limit is + /// not: the bound is the protocol's, not a guess. + #[test] + fn a_label_is_held_to_its_dns_length() { + let sixty_three = "a".repeat(63); + assert!(resolve_issuer_domain_name(&format!("{sixty_three}.org")).is_ok()); + assert!(resolve_issuer_domain_name(&format!("{sixty_three}a.org")).is_err()); + } + + /// A rejection has to say what to change: the character it objected to, not + /// just that the value was wrong. + #[test] + fn a_rejection_names_the_character_it_rejected() { + let err = resolve_issuer_domain_name("my_ca.example.com") + .expect_err("an underscore is not a DNS label character"); + assert!(err.to_string().contains('_'), "{err:#}"); + + // A multi-byte character has to name itself. Scanning bytes and casting + // to `char` would report the first byte of `é` as `Ã`, which does not + // occur in the value the same message quotes back. + let err = resolve_issuer_domain_name("lé.org").expect_err("é is not a DNS label character"); + assert!(err.to_string().contains('é'), "{err:#}"); + assert!(!err.to_string().contains('Ã'), "{err:#}"); + } + + /// Empty is what an untouched configuration carries, and it means the + /// default rather than a name of zero length -- which as an `issue-value` + /// would be CAA denying every issuer. + #[test] + fn an_unset_issuer_name_is_the_default_one() { + assert_eq!( + resolve_issuer_domain_name("").unwrap(), + LETS_ENCRYPT_ISSUER_DOMAIN_NAME + ); + assert_eq!( + resolve_issuer_domain_name(" ").unwrap(), + LETS_ENCRYPT_ISSUER_DOMAIN_NAME + ); + assert_eq!( + resolve_issuer_domain_name("pebble.letsencrypt.org.").unwrap(), + "pebble.letsencrypt.org" + ); + } + + /// The name is interpolated into records that a `;` or a space makes + /// malformed, and CAA is rewritten by deleting the old records first, so a + /// typo accepted here is a zone that forbids all issuance. + #[test] + fn a_name_that_would_corrupt_a_record_is_refused() { + for bad in [ + "lets encrypt.org", + "letsencrypt.org;policy=wildcard", + "#letsencrypt.org", + "letsencrypt..org", + "\"letsencrypt.org\"", + // RFC 8659's label grammar bounds both ends of a label, so these + // are as unparseable to a strict CA as a space is. + "_letsencrypt.org", + "-letsencrypt.org", + "letsencrypt-.org", + "lé.org", + ] { + assert!( + resolve_issuer_domain_name(bad).is_err(), + "{bad:?} should be refused" + ); + } + } + + #[test] + fn validation_domain_prepends_the_label() { + assert_eq!( + validation_domain("example.com"), + "_validation-persist.example.com" + ); + } + + #[test] + fn validation_domain_strips_the_wildcard_prefix() { + // The CA looks up the base name for a wildcard authorization, so a record + // at `_validation-persist.*.example.com` would never be read. + assert_eq!( + validation_domain("*.example.com"), + "_validation-persist.example.com" + ); + } + + #[test] + fn rdata_has_no_trailing_semicolon() { + assert_eq!( + record(false).rdata(), + format!("letsencrypt.org; accounturi={ACCOUNT}") + ); + } + + #[test] + fn rdata_carries_the_wildcard_policy() { + assert_eq!( + record(true).rdata(), + format!("letsencrypt.org; accounturi={ACCOUNT}; policy=wildcard") + ); + } + + #[test] + fn display_renders_a_zone_file_line() { + assert_eq!( + record(false).to_string(), + format!("IN TXT \"letsencrypt.org; accounturi={ACCOUNT}\"") + ); + } + + #[test] + fn rendered_record_parses_back() { + for wildcard in [false, true] { + assert!( + accepts(wildcard, &record(wildcard).rdata()), + "wildcard={wildcard}" + ); + } + } + + #[test] + fn accepts_the_draft_example_layout() { + // draft-ietf-acme-dns-persist-01, figure 2, plus the whitespace RFC 8659 + // allows around the issuer name and each parameter. + let record = AuthorizationRecord { + issuer_domain_name: "authority.example".to_string(), + account_uri: "https://ca.example/acct/123".to_string(), + wildcard: false, + }; + for published in [ + "authority.example; accounturi=https://ca.example/acct/123", + "authority.example;accounturi=https://ca.example/acct/123", + "\tauthority.example ;\taccounturi=https://ca.example/acct/123 ", + ] { + assert!(record.satisfied_by(published, NOW), "{published:?}"); + } + } + + #[test] + fn matches_tags_case_insensitively() { + assert!(accepts( + true, + &format!("letsencrypt.org; AccountURI={ACCOUNT}; Policy=WILDCARD") + )); + } + + #[test] + fn rejects_a_different_account_uri() { + assert!(!accepts( + false, + "letsencrypt.org; accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/9" + )); + } + + #[test] + fn compares_the_account_uri_without_case_folding() { + // The draft pins Simple String Comparison, so an upper-cased URI is a + // different account as far as the CA is concerned. + assert!(!accepts( + false, + &format!("letsencrypt.org; accounturi={}", ACCOUNT.to_uppercase()) + )); + } + + #[test] + fn ignores_a_record_naming_another_issuer() { + assert!(!accepts( + false, + &format!("otherca.example; accounturi={ACCOUNT}") + )); + } + + #[test] + fn rejects_a_plain_record_for_a_wildcard_request() { + let published = record(false).rdata(); + assert!(accepts(false, &published)); + assert!(!accepts(true, &published)); + } + + #[test] + fn accepts_a_wildcard_record_for_a_plain_request() { + assert!(accepts(false, &record(true).rdata())); + } + + #[test] + fn rejects_a_trailing_semicolon() { + // A CA reads the empty tail as an empty parameter and fails the whole + // record, so a record rendered with one would be silently unusable. + assert!(!accepts( + false, + &format!("letsencrypt.org; accounturi={ACCOUNT};") + )); + } + + #[test] + fn rejects_a_duplicate_parameter() { + assert!(!accepts( + false, + &format!("letsencrypt.org; accounturi={ACCOUNT}; accounturi={ACCOUNT}") + )); + } + + #[test] + fn rejects_whitespace_inside_a_value() { + assert!(!accepts( + false, + "letsencrypt.org; accounturi=https://ca.example/acct/1 2" + )); + } + + #[test] + fn rejects_a_record_without_accounturi() { + assert!(!accepts(false, "letsencrypt.org; policy=wildcard")); + } + + #[test] + fn ignores_unrecognized_parameters() { + assert!(accepts( + true, + &format!("letsencrypt.org; accounturi={ACCOUNT}; policy=wildcard; futuretag=whatever") + )); + } + + #[test] + fn rejects_an_expired_persist_until() { + let published = format!("letsencrypt.org; accounturi={ACCOUNT}; persistUntil={NOW}"); + assert!(record(false).satisfied_by(&published, NOW)); + assert!(!record(false).satisfied_by(&published, NOW + 1)); + } + + #[test] + fn rejects_a_malformed_persist_until() { + assert!(!accepts( + false, + &format!("letsencrypt.org; accounturi={ACCOUNT}; persistUntil=tomorrow") + )); + } + + #[test] + fn accepts_any_one_of_the_published_records() { + let published = vec![ + "otherca.example; accounturi=https://other.example/acct/1".to_string(), + record(true).rdata(), + ]; + assert!(record(true).satisfied_by_any(&published, NOW)); + assert!(!record(true).satisfied_by_any(&published[..1], NOW)); + } +} diff --git a/dstack/certbot/src/http_client.rs b/dstack/certbot/src/http_client.rs index 2de8f8232..f8aea21be 100644 --- a/dstack/certbot/src/http_client.rs +++ b/dstack/certbot/src/http_client.rs @@ -8,7 +8,7 @@ use anyhow::{Context, Result}; use bytes::Bytes; use http::Request; use http_body_util::{BodyExt, Full}; -use instant_acme::{BytesResponse, HttpClient}; +use instant_acme::{BodyWrapper, BytesResponse, HttpClient}; use reqwest::Client; use std::error::Error as StdError; use std::future::Future; @@ -35,7 +35,7 @@ impl ReqwestHttpClient { impl HttpClient for ReqwestHttpClient { fn request( &self, - req: Request<Full<Bytes>>, + req: Request<BodyWrapper<Bytes>>, ) -> Pin<Box<dyn Future<Output = Result<BytesResponse, instant_acme::Error>> + Send>> { let client = self.client.clone(); Box::pin(async move { diff --git a/dstack/certbot/src/lib.rs b/dstack/certbot/src/lib.rs index df71b9935..e2a0314c9 100644 --- a/dstack/certbot/src/lib.rs +++ b/dstack/certbot/src/lib.rs @@ -11,18 +11,24 @@ //! //! - Automatic certificate issuance and renewal //! - DNS-01 challenge support (currently implemented for Cloudflare) +//! - DNS-PERSIST-01 challenge support, which needs no DNS provider credential //! - Easy integration with existing Rust applications //! //! For more detailed information on the available methods and their usage, please refer //! to the documentation of individual structs and functions. -pub use acme_client::AcmeClient; +pub use acme_client::{ + advisory_dns_wait, required_dns_records, AcmeAccount, AcmeClient, ChallengeKind, + RequiredRecord, ValidationMethod, +}; pub use bot::{read_pubkey, CertBot, CertBotConfig}; pub use dns01_client::Dns01Client; +pub use dns_persist::{resolve_issuer_domain_name, LETS_ENCRYPT_ISSUER_DOMAIN_NAME}; pub use workdir::WorkDir; mod acme_client; mod bot; mod dns01_client; +mod dns_persist; mod http_client; mod workdir; diff --git a/dstack/certbot/src/workdir.rs b/dstack/certbot/src/workdir.rs index 9265834d1..690eeb65f 100644 --- a/dstack/certbot/src/workdir.rs +++ b/dstack/certbot/src/workdir.rs @@ -65,3 +65,50 @@ impl WorkDir { crate::bot::list_cert_public_keys(self.backup_dir()) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::symlink; + + #[test] + fn live_links_can_roll_back_to_a_complete_generation() { + let root = tempfile::tempdir().unwrap(); + let workdir = WorkDir::new(root.path()); + let first = workdir.backup_dir().join("0001"); + let second = workdir.backup_dir().join("0002"); + fs::create_dir_all(&first).unwrap(); + fs::create_dir_all(&second).unwrap(); + fs::write(first.join("cert.pem"), "first-cert").unwrap(); + fs::write(first.join("key.pem"), "first-key").unwrap(); + fs::write(second.join("cert.pem"), "second-cert").unwrap(); + fs::write(second.join("key.pem"), "second-key").unwrap(); + fs::create_dir_all(workdir.live_dir()).unwrap(); + symlink(second.join("cert.pem"), workdir.cert_path()).unwrap(); + symlink(second.join("key.pem"), workdir.key_path()).unwrap(); + fs::remove_file(workdir.cert_path()).unwrap(); + fs::remove_file(workdir.key_path()).unwrap(); + symlink(first.join("cert.pem"), workdir.cert_path()).unwrap(); + symlink(first.join("key.pem"), workdir.key_path()).unwrap(); + assert_eq!( + fs::read_to_string(workdir.cert_path()).unwrap(), + "first-cert" + ); + assert_eq!(fs::read_to_string(workdir.key_path()).unwrap(), "first-key"); + } + + #[test] + fn malformed_credentials_fail_without_mutating_the_archive() { + let root = tempfile::tempdir().unwrap(); + let workdir = WorkDir::new(root.path()); + let archive = workdir.backup_dir().join("0001"); + fs::create_dir_all(&archive).unwrap(); + fs::write(archive.join("cert.pem"), "stable").unwrap(); + fs::write(workdir.account_credentials_path(), "not-json").unwrap(); + assert!(workdir.acme_account_uri().is_err()); + assert_eq!( + fs::read_to_string(archive.join("cert.pem")).unwrap(), + "stable" + ); + } +} diff --git a/dstack/crates/api-auth/Cargo.toml b/dstack/crates/api-auth/Cargo.toml new file mode 100644 index 000000000..c3df2919d --- /dev/null +++ b/dstack/crates/api-auth/Cargo.toml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "dstack-api-auth" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +anyhow.workspace = true +base64.workspace = true +bcrypt.workspace = true +rocket.workspace = true +sha2.workspace = true +subtle.workspace = true diff --git a/dstack/crates/api-auth/README.md b/dstack/crates/api-auth/README.md new file mode 100644 index 000000000..2fef4a795 --- /dev/null +++ b/dstack/crates/api-auth/README.md @@ -0,0 +1,31 @@ +<!-- +SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> + +SPDX-License-Identifier: Apache-2.0 +--> + +# dstack-api-auth + +Shared authentication primitives for dstack HTTP and RPC administration APIs. + +HTTP integrations support: + +- `Authorization: Bearer <token>`; +- the shared `X-Admin-Token` header used by Gateway and VMM; +- `Authorization: Basic ...` backed by a bcrypt Apache htpasswd file; +- optional GET-only `?token=` links for the Gateway and VMM dashboards. + +Generate a compatible password file with: + +```bash +htpasswd -B -c /etc/dstack/admin.htpasswd admin +``` + +Gateway's existing token configuration and environment variables remain +supported. VMM's existing `[auth] enabled` and `tokens` fields remain supported; +when enabled, authentication now covers the complete externally listening +Rocket server. The separate VMM host-vsock server is intentionally unaffected. + +KMS continues to accept the existing SHA-256 `admin_token_hash` and protobuf +request token. Its comparison uses the constant-time verifier from this crate; +the public KMS application APIs are not placed behind operator HTTP Basic auth. diff --git a/dstack/crates/api-auth/src/lib.rs b/dstack/crates/api-auth/src/lib.rs new file mode 100644 index 000000000..c52f7bcb3 --- /dev/null +++ b/dstack/crates/api-auth/src/lib.rs @@ -0,0 +1,457 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Shared HTTP API authentication for dstack services. +//! +//! Supports bearer/shared-secret tokens, HTTP Basic credentials backed by an +//! Apache htpasswd file, and an optional GET-only query token for compatibility +//! with browser dashboard links. + +use std::{collections::HashMap, path::Path, sync::Arc}; + +use anyhow::{Context, Result}; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; +use rocket::{ + fairing::{Fairing, Info, Kind}, + http::{uri::Origin, Header, Method, Status}, + response::Responder, + Data, Request, Response, Route, +}; +use sha2::{Digest, Sha256}; +use subtle::{Choice, ConstantTimeEq}; + +const UNAUTH_URI: &str = "/__dstack_api_auth_unauthorized"; + +#[derive(Debug)] +struct Htpasswd { + entries: HashMap<String, String>, +} + +impl Htpasswd { + fn load(path: &Path) -> Result<Self> { + let contents = std::fs::read_to_string(path) + .with_context(|| format!("failed to read htpasswd file {}", path.display()))?; + Self::parse(&contents) + .with_context(|| format!("failed to parse htpasswd file {}", path.display())) + } + + fn parse(contents: &str) -> Result<Self> { + let mut entries = HashMap::new(); + for (line_number, line) in contents.lines().enumerate() { + if line.trim().is_empty() || line.starts_with('#') { + continue; + } + + let (username, hash) = line + .split_once(':') + .with_context(|| format!("invalid entry on line {}", line_number + 1))?; + if username.is_empty() { + anyhow::bail!("empty username on line {}", line_number + 1); + } + if !matches!(hash.get(..4), Some("$2a$") | Some("$2b$") | Some("$2y$")) { + anyhow::bail!( + "unsupported hash for user {username:?} on line {}; only bcrypt hashes are supported", + line_number + 1 + ); + } + entries.insert(username.to_owned(), hash.to_owned()); + } + + if entries.is_empty() { + anyhow::bail!("file contains no users"); + } + Ok(Self { entries }) + } + + fn verify(&self, username: &str, password: &str) -> bool { + self.entries + .get(username) + .is_some_and(|hash| bcrypt::verify(password, hash).unwrap_or(false)) + } +} + +#[derive(Clone, Default)] +pub struct Authenticator { + enabled: bool, + token_hashes: Arc<Vec<[u8; 32]>>, + htpasswd: Option<Arc<Htpasswd>>, +} + +impl Authenticator { + pub fn disabled() -> Self { + Self::default() + } + + pub fn from_tokens(tokens: impl IntoIterator<Item = impl AsRef<str>>) -> Self { + Self { + enabled: true, + token_hashes: Arc::new( + tokens + .into_iter() + .filter(|token| !token.as_ref().is_empty()) + .map(|token| sha256(token.as_ref().as_bytes())) + .collect(), + ), + htpasswd: None, + } + } + + pub fn from_token_hashes(hashes: Vec<[u8; 32]>) -> Self { + Self { + enabled: true, + token_hashes: Arc::new(hashes), + htpasswd: None, + } + } + + pub fn with_htpasswd_file(mut self, path: impl AsRef<Path>) -> Result<Self> { + self.enabled = true; + let path = path.as_ref(); + self.htpasswd = Some(Arc::new(Htpasswd::load(path)?)); + Ok(self) + } + + pub fn is_enabled(&self) -> bool { + self.enabled + } + + pub fn verify_token(&self, token: &str) -> bool { + let candidate = sha256(token.as_bytes()); + let matched = self + .token_hashes + .iter() + .fold(Choice::from(0), |matched, expected| { + matched | candidate.ct_eq(expected) + }); + bool::from(matched) + } + + pub fn verify_basic(&self, username: &str, password: &str) -> bool { + self.htpasswd + .as_ref() + .is_some_and(|htpasswd| htpasswd.verify(username, password)) + // Preserve the old dashboard behavior: Basic user:token and + // token: are aliases for a shared token. + || self.verify_token(password) + || (password.is_empty() && self.verify_token(username)) + } +} + +fn sha256(bytes: &[u8]) -> [u8; 32] { + Sha256::digest(bytes).into() +} + +/// Verify a plaintext token against the legacy wire/config representation of +/// a SHA-256 digest. Invalid digest lengths simply fail authentication. +pub fn verify_sha256_token(token: &str, expected: &[u8]) -> bool { + let candidate = sha256(token.as_bytes()); + expected.len() == candidate.len() && bool::from(candidate.as_slice().ct_eq(expected)) +} + +#[derive(Clone)] +pub struct HttpAuthConfig { + pub realm: String, + pub token_header: Option<String>, + pub allow_get_query_token: bool, +} + +impl Default for HttpAuthConfig { + fn default() -> Self { + Self { + realm: "dstack API".into(), + token_header: None, + allow_get_query_token: false, + } + } +} + +pub struct HttpAuthFairing { + authenticator: Authenticator, + config: HttpAuthConfig, +} + +impl HttpAuthFairing { + pub fn new(authenticator: Authenticator, config: HttpAuthConfig) -> Self { + Self { + authenticator, + config, + } + } + + fn authorized(&self, req: &Request<'_>) -> bool { + if !self.authenticator.is_enabled() { + return true; + } + if let Some(header) = &self.config.token_header { + if req + .headers() + .get_one(header) + .is_some_and(|token| self.authenticator.verify_token(token)) + { + return true; + } + } + if let Some(value) = req.headers().get_one("Authorization") { + if let Some(token) = value.strip_prefix("Bearer ") { + return self.authenticator.verify_token(token.trim()); + } + if let Some(encoded) = value.strip_prefix("Basic ") { + if let Some((username, password)) = decode_basic(encoded.trim()) { + return self.authenticator.verify_basic(&username, &password); + } + } + } + self.config.allow_get_query_token + && req.method() == Method::Get + && req.query_fields().any(|field| { + field.name.key_lossy().as_str() == "token" + && self.authenticator.verify_token(field.value.as_ref()) + }) + } +} + +fn decode_basic(encoded: &str) -> Option<(String, String)> { + let decoded = BASE64.decode(encoded).ok()?; + let text = std::str::from_utf8(&decoded).ok()?; + let (username, password) = text.split_once(':').unwrap_or((text, "")); + Some((username.to_owned(), password.to_owned())) +} + +fn strip_token_query(uri: &Origin<'_>) -> Option<Origin<'static>> { + let mut found = false; + let kept: Vec<_> = uri + .query()? + .as_str() + .split('&') + .filter(|pair| { + let remove = pair.split('=').next() == Some("token"); + found |= remove; + !remove && !pair.is_empty() + }) + .collect(); + found.then(|| { + let path = uri.path().as_str(); + Origin::parse_owned(if kept.is_empty() { + path.to_owned() + } else { + format!("{path}?{}", kept.join("&")) + }) + .ok() + })? +} + +#[rocket::async_trait] +impl Fairing for HttpAuthFairing { + fn info(&self) -> Info { + Info { + name: "dstack API authentication", + kind: Kind::Request, + } + } + + async fn on_request(&self, req: &mut Request<'_>, _: &mut Data<'_>) { + req.local_cache(|| RequestRealm(self.config.realm.clone())); + if req.uri().path() == UNAUTH_URI { + return; + } + if !self.authorized(req) { + if let Ok(uri) = Origin::parse_owned(UNAUTH_URI.to_owned()) { + req.set_uri(uri); + } + } else if self.config.allow_get_query_token { + if let Some(uri) = strip_token_query(req.uri()) { + req.set_uri(uri); + } + } + } +} + +struct Unauthorized; + +impl<'r> Responder<'r, 'static> for Unauthorized { + fn respond_to(self, req: &'r Request<'_>) -> rocket::response::Result<'static> { + let realm = &req.local_cache(|| RequestRealm("dstack API".into())).0; + Response::build() + .status(Status::Unauthorized) + .header(Header::new( + "WWW-Authenticate", + format!("Basic realm=\"{realm}\""), + )) + .ok() + } +} + +struct RequestRealm(String); + +#[rocket::get("/__dstack_api_auth_unauthorized")] +fn unauth_get() -> Unauthorized { + Unauthorized +} +#[rocket::post("/__dstack_api_auth_unauthorized", data = "<_data>")] +fn unauth_post(_data: Data<'_>) -> Unauthorized { + Unauthorized +} +#[rocket::put("/__dstack_api_auth_unauthorized", data = "<_data>")] +fn unauth_put(_data: Data<'_>) -> Unauthorized { + Unauthorized +} +#[rocket::patch("/__dstack_api_auth_unauthorized", data = "<_data>")] +fn unauth_patch(_data: Data<'_>) -> Unauthorized { + Unauthorized +} +#[rocket::delete("/__dstack_api_auth_unauthorized")] +fn unauth_delete() -> Unauthorized { + Unauthorized +} +#[rocket::options("/__dstack_api_auth_unauthorized")] +fn unauth_options() -> Unauthorized { + Unauthorized +} +#[rocket::head("/__dstack_api_auth_unauthorized")] +fn unauth_head() -> Unauthorized { + Unauthorized +} + +pub fn routes() -> Vec<Route> { + rocket::routes![ + unauth_get, + unauth_post, + unauth_put, + unauth_patch, + unauth_delete, + unauth_options, + unauth_head + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use rocket::{http::Header, local::asynchronous::Client}; + + #[rocket::get("/ok")] + fn ok() -> &'static str { + "ok" + } + + #[rocket::post("/ok")] + fn post_ok() -> &'static str { + "ok" + } + + #[rocket::get("/echo?<token>&<other>")] + fn echo(token: Option<&str>, other: Option<&str>) -> String { + format!("{}:{}", token.unwrap_or("none"), other.unwrap_or("none")) + } + + async fn client(auth: Authenticator) -> Client { + Client::tracked( + rocket::build() + .attach(HttpAuthFairing::new( + auth, + HttpAuthConfig { + realm: "test realm".into(), + token_header: Some("X-Admin-Token".into()), + allow_get_query_token: true, + }, + )) + .mount("/", rocket::routes![ok, post_ok, echo]) + .mount("/", routes()), + ) + .await + .unwrap() + } + + #[test] + fn verifies_tokens_and_apache_hashes() { + let dir = std::env::temp_dir().join(format!("dstack-api-auth-{}", std::process::id())); + let password = format!("test-password-{}", std::process::id()); + let hash = bcrypt::hash(&password, 4).unwrap(); + std::fs::write(&dir, format!("alice:{hash}\n")).unwrap(); + let auth = Authenticator::from_tokens(["secret"]) + .with_htpasswd_file(&dir) + .unwrap(); + assert!(auth.verify_token("secret")); + assert!(!auth.verify_token("wrong")); + assert!(auth.verify_basic("alice", &password)); + assert!(!auth.verify_basic("alice", &format!("{password}-wrong"))); + let _ = std::fs::remove_file(dir); + } + + #[rocket::async_test] + async fn protects_all_methods_and_accepts_compatible_credentials() { + let client = client(Authenticator::from_tokens(["secret"])).await; + let response = client.get("/ok").dispatch().await; + assert_eq!(response.status(), Status::Unauthorized); + assert_eq!( + response.headers().get_one("WWW-Authenticate"), + Some("Basic realm=\"test realm\"") + ); + + assert_eq!( + client + .get("/ok") + .header(Header::new("Authorization", "Bearer secret")) + .dispatch() + .await + .status(), + Status::Ok + ); + assert_eq!( + client + .get("/ok") + .header(Header::new("X-Admin-Token", "secret")) + .dispatch() + .await + .status(), + Status::Ok + ); + assert_eq!( + client.get("/ok?token=secret").dispatch().await.status(), + Status::Ok + ); + assert_eq!( + client.post("/ok?token=secret").dispatch().await.status(), + Status::Unauthorized + ); + assert_eq!( + client + .get("/ok") + .header(Header::new( + "Authorization", + format!("Basic {}", BASE64.encode("admin:secret")), + )) + .dispatch() + .await + .status(), + Status::Ok + ); + assert_eq!( + client + .put("/ok") + .header(Header::new("Authorization", "Bearer wrong")) + .dispatch() + .await + .status(), + Status::Unauthorized + ); + let body = client + .get("/echo?token=secret&other=keep") + .dispatch() + .await + .into_string() + .await + .unwrap(); + assert_eq!(body, "none:keep"); + } + + #[rocket::async_test] + async fn enabled_with_no_credentials_denies_access() { + let client = client(Authenticator::from_tokens(Vec::<String>::new())).await; + assert_eq!( + client.get("/ok").dispatch().await.status(), + Status::Unauthorized + ); + } +} diff --git a/dstack/crates/build-info/Cargo.toml b/dstack/crates/build-info/Cargo.toml new file mode 100644 index 000000000..ae53b0965 --- /dev/null +++ b/dstack/crates/build-info/Cargo.toml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "dstack-build-info" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +git-version = "0.3.9" diff --git a/dstack/crates/build-info/src/lib.rs b/dstack/crates/build-info/src/lib.rs new file mode 100644 index 000000000..2400cc018 --- /dev/null +++ b/dstack/crates/build-info/src/lib.rs @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Common compile-time build information for dstack binaries. + +#[doc(hidden)] +pub use git_version::git_version as __git_version; + +/// Returns the current Git commit as `git:<hash>`. +/// +/// Tags are deliberately excluded because dstack is a monorepo containing +/// component-specific tags. Letting `git describe` select the nearest tag can +/// therefore report an unrelated component's version. +/// +/// Guest OS backends build the workspace without a `.git` directory in the +/// sandbox, so `DSTACK_BUILD_GIT_REVISION` overrides the value at compile time. +/// It must carry the full display string including the `git:` prefix, because +/// callers such as the guest agent's `Info.rev` field surface it verbatim. +/// The name is deliberately distinct from the Yocto backend's +/// `DSTACK_GIT_REVISION`, which carries a bare SHA for release metadata and +/// would otherwise be picked up here with the wrong format. +#[macro_export] +macro_rules! git_revision { + () => { + match option_env!("DSTACK_BUILD_GIT_REVISION") { + Some(revision) => revision, + None => $crate::__git_version!( + args = [ + "--abbrev=20", + "--always", + "--dirty=-modified", + "--exclude=*" + ], + prefix = "git:", + fallback = "unknown" + ), + } + }; +} + +/// Returns the calling package's version and Git revision for display. +/// +/// The result has the form `v0.6.0 (git:0123456789abcdef0123)`. This is a +/// macro so `CARGO_PKG_VERSION` is evaluated for the calling package rather +/// than for `dstack-build-info` itself. +#[macro_export] +macro_rules! app_version { + () => { + format!( + "v{} ({})", + env!("CARGO_PKG_VERSION"), + $crate::git_revision!() + ) + }; +} + +#[cfg(test)] +mod tests { + #[test] + fn git_revision_is_never_empty() { + assert!(!crate::git_revision!().is_empty()); + } + + #[test] + fn app_version_embeds_package_version_and_revision() { + let version = crate::app_version!(); + assert!(version.starts_with(&format!("v{}", env!("CARGO_PKG_VERSION")))); + assert!(version.ends_with(&format!("({})", crate::git_revision!()))); + } + + #[test] + fn build_override_replaces_the_git_derived_revision() { + // The override is read at compile time, so this asserts the contract + // that applies to whichever of the two arms was expanded. + match option_env!("DSTACK_BUILD_GIT_REVISION") { + Some(revision) => assert_eq!(crate::git_revision!(), revision), + None => assert!(crate::git_revision!().starts_with("git:")), + } + } +} diff --git a/dstack/crates/dstack-auth/src/main.rs b/dstack/crates/dstack-auth/src/main.rs index 217d92947..2a752e4b0 100644 --- a/dstack/crates/dstack-auth/src/main.rs +++ b/dstack/crates/dstack-auth/src/main.rs @@ -291,7 +291,7 @@ mod tests { #[test] fn deserializes_the_kms_bootinfo_wire_contract() { let wire = r#"{ - "attestationMode": "dstack", + "teeVariant": "dstack", "mrAggregated": "0xAABB", "osImageHash": "0xC2AA", "mrSystem": "0xdead", diff --git a/dstack/crates/dstack-cli-core/Cargo.toml b/dstack/crates/dstack-cli-core/Cargo.toml index ea74358bc..3f4d0bdcc 100644 --- a/dstack/crates/dstack-cli-core/Cargo.toml +++ b/dstack/crates/dstack-cli-core/Cargo.toml @@ -12,7 +12,9 @@ license.workspace = true anyhow.workspace = true http-client = { workspace = true, features = ["prpc"] } dstack-vmm-rpc.workspace = true +dstack-types.workspace = true serde_json.workspace = true +safe-write.workspace = true # advisory file locking (flock) for the allowlist/state read-modify-write; # already in the dependency tree transitively, so no extra compile cost. rustix = { version = "0.38", features = ["fs"] } diff --git a/dstack/crates/dstack-cli-core/src/compose.rs b/dstack/crates/dstack-cli-core/src/compose.rs index 5c395213d..4db4f53a6 100644 --- a/dstack/crates/dstack-cli-core/src/compose.rs +++ b/dstack/crates/dstack-cli-core/src/compose.rs @@ -13,10 +13,47 @@ use serde_json::json; /// `kms_enabled` selects KMS mode (deterministic, upgradeable per-app keys); /// gateway and local-key-provider are off for the direct-port single-node flow. pub fn build_app_compose(name: &str, docker_compose_yaml: &str, kms_enabled: bool) -> String { - let manifest = json!({ - "manifest_version": 2, + build_app_compose_with_runtime( + name, + docker_compose_yaml, + kms_enabled, + "docker-compose", + None, + ) +} + +/// Build an app-compose manifest with an explicitly selected compose frontend. +/// `snapshotter` is meaningful only for `nerdctl-compose`. +pub fn build_app_compose_with_runtime( + name: &str, + docker_compose_yaml: &str, + kms_enabled: bool, + runner: &str, + snapshotter: Option<&str>, +) -> String { + build_app_compose_with_runtime_and_volumes( + name, + docker_compose_yaml, + kms_enabled, + runner, + snapshotter, + &[], + ) +} + +/// Build an app-compose manifest with measured verity volume declarations. +pub fn build_app_compose_with_runtime_and_volumes( + name: &str, + docker_compose_yaml: &str, + kms_enabled: bool, + runner: &str, + snapshotter: Option<&str>, + verity_volumes: &[dstack_types::VerityVolume], +) -> String { + let mut manifest = json!({ + "manifest_version": if runner == "nerdctl-compose" { json!("3") } else { json!(2) }, "name": name, - "runner": "docker-compose", + "runner": runner, "docker_compose_file": docker_compose_yaml, "kms_enabled": kms_enabled, "gateway_enabled": false, @@ -31,6 +68,12 @@ pub fn build_app_compose(name: &str, docker_compose_yaml: &str, kms_enabled: boo // (NTS is also currently broken in guest images — see dstack#745.) "secure_time": false, }); + if let Some(snapshotter) = snapshotter { + manifest["snapshotter"] = json!(snapshotter); + } + if !verity_volumes.is_empty() { + manifest["verity_volumes"] = json!(verity_volumes); + } // pretty-print via Value's Display (`{:#}`) — infallible, and byte-identical // to serde_json::to_string_pretty (avoids an expect on an unfailable Result). format!("{manifest:#}") diff --git a/dstack/crates/dstack-cli-core/src/config.rs b/dstack/crates/dstack-cli-core/src/config.rs index a0ca82417..4309bd9bb 100644 --- a/dstack/crates/dstack-cli-core/src/config.rs +++ b/dstack/crates/dstack-cli-core/src/config.rs @@ -9,7 +9,8 @@ //! false` + a set `auto_bootstrap_domain`, the combination validated to make //! bootstrap hands-off). //! * `auth-allowlist.json` — read by the host-side Rust auth webhook. -//! * `vmm.toml` — the host VMM config (gateway + auth-token gating off). +//! * `vmm.toml` — the host VMM config (gateway off; management-API auth token +//! set by `dstackup install`). use crate::host::Platform; use anyhow::{Context, Result}; @@ -129,13 +130,8 @@ port = 8000 key = "/kms/certs/rpc.key" certs = "/kms/certs/rpc.crt" -[rpc.tls.mutual] -ca_certs = "/kms/certs/tmp-ca.crt" -mandatory = false - [core] cert_dir = "/kms/certs" -admin_token_hash = "" # single-node: the KMS does not self-attest to its own auth API before # bootstrap (it still attests the genesis keys via the guest agent, and app # auth + per-app quote checks are unaffected). @@ -278,6 +274,11 @@ pub struct VmmRender { pub kms_urls: Vec<String>, /// confidential-computing platform (selects qemu/share-mode for the CVMs). pub platform: Platform, + /// gate the management API behind a bearer/Basic token (`[auth] enabled`). + pub auth_enabled: bool, + /// the token accepted by the management API when `auth_enabled` is set. + /// Empty renders `tokens = []`. + pub auth_token: String, } impl Default for VmmRender { @@ -296,13 +297,16 @@ impl Default for VmmRender { key_provider_port: 3443, kms_urls: Vec::new(), platform: Platform::Tdx, + auth_enabled: false, + auth_token: String::new(), } } } -/// render the host `vmm.toml`. Gateway and auth-token gating are off -/// (single-node direct-port access); CVMs use user-mode networking with host -/// port mapping. +/// render the host `vmm.toml`. Gateway is off (single-node direct-port +/// access); management-API auth is gated by `r.auth_enabled`/`r.auth_token` +/// (`dstackup install` generates a token and enables it). CVMs use user-mode +/// networking with host port mapping. pub fn vmm_toml(r: &VmmRender) -> String { format!( r#"# generated by `dstackup install` @@ -352,7 +356,6 @@ mode = "user" net = "10.0.2.0/24" dhcp_start = "10.0.2.10" restrict = false -forward_service_enabled = false [cvm.port_mapping] enabled = true @@ -377,9 +380,12 @@ base_domain = "localhost" port = 8082 agent_port = 8090 +# management API auth. `dstackup install` generates a token and enables this +# so the VMM control surface (create/stop VM, UI, pRPC) is not exposed +# unauthenticated. Clients send `Authorization: Bearer <token>`. [auth] -enabled = false -tokens = [] +enabled = {auth_enabled} +tokens = [{auth_tokens}] [supervisor] exe = "{supervisor_exe}" @@ -424,6 +430,12 @@ port = {kp_port} host_api_port = r.host_api_port, kp_addr = r.key_provider_addr, kp_port = r.key_provider_port, + auth_enabled = r.auth_enabled, + auth_tokens = if r.auth_token.is_empty() { + String::new() + } else { + format!("\"{}\"", r.auth_token) + }, ) } @@ -437,15 +449,28 @@ mod tests { dashboard_addr: "tcp:127.0.0.1:19080".into(), cid_start: 2000, host_api_port: 10001, + auth_enabled: true, + auth_token: "deadbeef".into(), ..Default::default() }; let rendered = vmm_toml(&r); assert!(rendered.contains(r#"address = "tcp:127.0.0.1:19080""#)); assert!(rendered.contains("cid_start = 2000")); assert!(rendered.contains("port = 10001")); + assert!(rendered.contains("enabled = true")); + assert!(rendered.contains(r#"tokens = ["deadbeef"]"#)); toml::from_str::<toml::Value>(&rendered).expect("vmm.toml must be valid TOML"); } + #[test] + fn vmm_toml_auth_disabled_renders_empty_tokens() { + let rendered = vmm_toml(&VmmRender::default()); + // default (no token) must still be valid TOML with an empty token list. + assert!(rendered.contains("tokens = []")); + let v: toml::Value = toml::from_str(&rendered).expect("vmm.toml must be valid TOML"); + assert_eq!(v["auth"]["enabled"].as_bool(), Some(false)); + } + #[test] fn kms_toml_has_single_node_invariants() { let cfg = HostConfig { diff --git a/dstack/crates/dstack-cli-core/src/fsutil.rs b/dstack/crates/dstack-cli-core/src/fsutil.rs index e03567e0a..8db04d990 100644 --- a/dstack/crates/dstack-cli-core/src/fsutil.rs +++ b/dstack/crates/dstack-cli-core/src/fsutil.rs @@ -10,44 +10,58 @@ //! auth webhook fails *closed* on invalid JSON, so a half-written allowlist //! denies keys to every app on the host. These helpers make the write atomic //! and serialize concurrent writers. +//! +//! The atomic replace itself is delegated to the `safe-write` crate, which the +//! rest of the workspace already uses. The two concerns stay separate: +//! `safe-write` makes a single write atomic and durable, while +//! [`lock_exclusive`] is what serializes a *read*-modify-write so two processes +//! cannot each publish a complete file built from the same stale read. use anyhow::{Context, Result}; use std::ffi::OsString; use std::fs::{File, OpenOptions}; -use std::io::Write; use std::path::{Path, PathBuf}; /// `path` with `suffix` appended to its full name (not replacing the extension, -/// so `a/b.json` + `.tmp` → `a/b.json.tmp`, a sibling in the same directory). +/// so `a/b.json` + `.lock` → `a/b.json.lock`, a sibling in the same directory). fn sibling(path: &Path, suffix: &str) -> PathBuf { let mut s: OsString = path.as_os_str().to_os_string(); s.push(suffix); PathBuf::from(s) } -/// atomically replace `path`'s contents: write a sibling temp file, fsync it, -/// rename it over the target, then fsync the directory. A reader (or a crash) -/// sees either the old file or the new one, never a fragment, and the rename is -/// durable across a power loss. `tmp` and `path` are in the same directory so -/// the rename is atomic. +/// atomically replace `path`'s contents: write a uniquely named temp file in +/// the same directory, fsync it, rename it over the target, then fsync the +/// directory. A reader (or a crash) sees either the old file or the new one, +/// never a fragment, and the rename is durable across a power loss. +/// +/// Concurrent writers to the same path all succeed and each publishes its +/// complete content; the winner is whoever renames last. That is *not* a +/// substitute for [`lock_exclusive`] when the write is part of a +/// read-modify-write — see the module docs. pub fn write_atomic(path: &Path, contents: &str) -> Result<()> { - let tmp = sibling(path, ".tmp"); - let mut f = - File::create(&tmp).with_context(|| format!("creating temp file {}", tmp.display()))?; - f.write_all(contents.as_bytes()) - .with_context(|| format!("writing {}", tmp.display()))?; - f.sync_all() - .with_context(|| format!("syncing {}", tmp.display()))?; - drop(f); - std::fs::rename(&tmp, path) - .with_context(|| format!("renaming {} -> {}", tmp.display(), path.display()))?; - // fsync the containing directory so the rename itself survives a crash. - if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) { - if let Ok(d) = File::open(dir) { - let _ = d.sync_all(); - } + safe_write::safe_write(path, contents).with_context(|| format!("writing {}", path.display())) +} + +/// like [`write_atomic`], but the file is created with `mode` (Unix permission +/// bits) *before* any content is written, so a secret never exists on disk with +/// broader-than-intended permissions — not even transiently between the rename +/// and a follow-up `chmod`. Use for credential files (`0o600`). +/// +/// `mode` is subject to the process umask, matching +/// [`std::os::unix::fs::OpenOptionsExt::mode`], so the result is never wider +/// than requested. On non-Unix platforms `mode` is ignored, as before. +pub fn write_atomic_mode(path: &Path, contents: &str, mode: u32) -> Result<()> { + #[cfg(unix)] + { + safe_write::safe_write_with_mode(path, contents, mode) + .with_context(|| format!("writing {}", path.display())) + } + #[cfg(not(unix))] + { + let _ = mode; + write_atomic(path, contents) } - Ok(()) } /// acquire an exclusive advisory lock tied to `path` (held on a sibling @@ -72,17 +86,91 @@ pub fn lock_exclusive(path: &Path) -> Result<File> { mod tests { use super::*; + /// every entry in `dir`, so "no temp file left behind" can be asserted + /// without knowing the temp file's name. + fn entries(dir: &Path) -> Vec<String> { + let mut v: Vec<String> = std::fs::read_dir(dir) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + v.sort(); + v + } + #[test] fn atomic_write_replaces_contents() { let dir = std::env::temp_dir().join(format!("dstack-fsutil-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).unwrap(); let p = dir.join("x.json"); write_atomic(&p, "one").unwrap(); assert_eq!(std::fs::read_to_string(&p).unwrap(), "one"); write_atomic(&p, "two").unwrap(); assert_eq!(std::fs::read_to_string(&p).unwrap(), "two"); - // no temp file left behind. - assert!(!sibling(&p, ".tmp").exists()); + // no temp file left behind, whatever it was called. + assert_eq!(entries(&dir), vec!["x.json".to_string()]); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Two writers racing on one path must both succeed, and the result must be + /// exactly one of them — not a mixture, and not a spurious failure. The + /// previous hand-rolled implementation used a fixed `<path>.tmp`, so + /// writers truncated each other's temp file and 3 in 4 writes failed. + #[test] + fn concurrent_writers_do_not_clobber_each_other() { + const LEN: usize = 256 * 1024; + let dir = std::env::temp_dir().join(format!("dstack-fsrace-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join("state.json"); + + for _ in 0..10 { + std::thread::scope(|s| { + for c in ['a', 'b', 'c', 'd'] { + let p = &p; + s.spawn(move || { + let body: String = std::iter::repeat_n(c, LEN).collect(); + write_atomic(p, &body).expect("concurrent write must not fail"); + }); + } + }); + let got = std::fs::read_to_string(&p).unwrap(); + assert_eq!(got.len(), LEN, "torn write: wrong length"); + let distinct: std::collections::BTreeSet<char> = got.chars().collect(); + assert_eq!(distinct.len(), 1, "torn write: mixed two writers' content"); + } + assert_eq!(entries(&dir), vec!["state.json".to_string()]); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Rewriting a credential file through the plain helper must not widen its + /// permissions. The previous implementation reset them to `0o666 & !umask`. + #[cfg(unix)] + #[test] + fn rewrite_preserves_existing_permissions() { + use std::os::unix::fs::PermissionsExt; + let dir = std::env::temp_dir().join(format!("dstack-fsperm-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join("token"); + write_atomic_mode(&p, "secret", 0o600).unwrap(); + write_atomic(&p, "rotated").unwrap(); + let mode = std::fs::metadata(&p).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600, "rewrite widened a credential file to {mode:o}"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[cfg(unix)] + #[test] + fn atomic_write_mode_creates_owner_only_file() { + use std::os::unix::fs::PermissionsExt; + let dir = std::env::temp_dir().join(format!("dstack-fsmode-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join("token"); + write_atomic_mode(&p, "secret", 0o600).unwrap(); + assert_eq!(std::fs::read_to_string(&p).unwrap(), "secret"); + let mode = std::fs::metadata(&p).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600, "credential file must be 0600, got {mode:o}"); let _ = std::fs::remove_dir_all(&dir); } diff --git a/dstack/crates/dstack-cli-core/src/ports.rs b/dstack/crates/dstack-cli-core/src/ports.rs index ae662442a..142be1b16 100644 --- a/dstack/crates/dstack-cli-core/src/ports.rs +++ b/dstack/crates/dstack-cli-core/src/ports.rs @@ -33,7 +33,25 @@ pub fn tcp_port_free(addr: &str, port: u16) -> bool { /// * `<host>:<vm>` — tcp, 127.0.0.1 /// * `<proto>:<host>:<vm>` /// * `<proto>:<addr>:<host>:<vm>` +/// +/// Any of them may carry a trailing `@<nic>` naming which NIC the traffic +/// enters through, as `vmm-cli.py --port` does. Without it the VMM picks: the +/// first user-mode NIC, else the first bridge NIC. A single-NIC VM never needs +/// it. pub fn parse_port(spec: &str) -> Result<PortMapping> { + let (spec, nic_index) = match spec.rsplit_once('@') { + // Only digits. `parse()` would take " 1" and "+1" as 1, and a NIC index + // is a position in a list the caller wrote. + Some((rest, nic)) if nic.bytes().all(|byte| byte.is_ascii_digit()) && !nic.is_empty() => ( + rest, + Some( + nic.parse::<u32>() + .with_context(|| format!("invalid NIC index in --port '{spec}'"))?, + ), + ), + Some((_, nic)) => bail!("invalid NIC index in --port '{spec}': '{nic}' is not a number"), + None => (spec, None), + }; let parts: Vec<&str> = spec.split(':').collect(); let (proto, addr, host, vm) = match parts.as_slice() { [vm] => ("tcp", "127.0.0.1", "auto", *vm), @@ -58,6 +76,7 @@ pub fn parse_port(spec: &str) -> Result<PortMapping> { host_address: addr.to_string(), host_port, vm_port, + nic_index, }) } @@ -90,4 +109,23 @@ mod tests { assert!(parse_port("70000:80").is_err()); assert!(parse_port("8080:0").is_err()); } + + #[test] + fn a_mapping_can_name_the_nic_it_enters_through() { + assert_eq!(parse_port("8080:80").unwrap().nic_index, None); + + let pinned = parse_port("udp:0.0.0.0:7483:51820@1").unwrap(); + assert_eq!(pinned.nic_index, Some(1)); + assert_eq!(pinned.protocol, "udp"); + assert_eq!(pinned.host_address, "0.0.0.0"); + assert_eq!(pinned.host_port, 7483); + assert_eq!(pinned.vm_port, 51820); + + // Unpinned must stay unpinned rather than default to NIC 0: the VMM + // resolves it to the first user-mode NIC, which need not be the first. + assert_eq!(parse_port("8080:80@0").unwrap().nic_index, Some(0)); + for bad in ["8080:80@", "8080:80@ 1", "8080:80@+1", "8080:80@a"] { + assert!(parse_port(bad).is_err(), "{bad} must be refused"); + } + } } diff --git a/dstack/crates/dstack-cli-core/src/vmm.rs b/dstack/crates/dstack-cli-core/src/vmm.rs index 5703ee71b..cbec5d7d3 100644 --- a/dstack/crates/dstack-cli-core/src/vmm.rs +++ b/dstack/crates/dstack-cli-core/src/vmm.rs @@ -11,7 +11,7 @@ use anyhow::{anyhow, bail, Result}; use dstack_vmm_rpc::vmm_client::VmmClient; use dstack_vmm_rpc::{Id, StatusRequest, StatusResponse, VmConfiguration}; -use http_client::http_request; +use http_client::http_request_with_headers; use http_client::prpc::PrpcClient; /// default local VMM control socket (created by `dstackup install`). @@ -20,25 +20,46 @@ pub const DEFAULT_HOST: &str = "unix:/var/run/dstack/vmm.sock"; /// a connection to a VMM — local unix socket or remote http endpoint. pub struct Vmm { rpc: VmmClient<PrpcClient>, - /// base string usable with [`http_request`] for non-prpc endpoints. + /// base string usable with [`http_request_with_headers`] for non-prpc endpoints. base: String, + /// bearer token sent with every request when the VMM has `[auth]` enabled. + auth_token: Option<String>, } impl Vmm { /// connect to a VMM addressed by `host`: /// `unix:/path/to/vmm.sock` (local) or `http(s)://host:port` (remote). pub fn connect(host: &str) -> Result<Self> { + Self::connect_with_token(host, None) + } + + /// like [`Vmm::connect`], additionally sending `Authorization: Bearer + /// <token>` with every request — required when the VMM has `[auth]` + /// enabled. An empty or `None` token sends no header. + pub fn connect_with_token(host: &str, token: Option<&str>) -> Result<Self> { let host = host.trim(); + let token = token.map(str::trim).filter(|t| !t.is_empty()); + let with_token = |client: PrpcClient| match token { + Some(token) => client.with_bearer_token(token), + None => client, + }; if let Some(sock) = host.strip_prefix("unix:") { - let rpc = VmmClient::new(PrpcClient::new_unix(sock.to_string(), "/prpc".to_string())); + let client = PrpcClient::new_unix(sock.to_string(), "/prpc".to_string()); + let rpc = VmmClient::new(with_token(client)); Ok(Self { rpc, base: format!("unix:{sock}"), + auth_token: token.map(str::to_string), }) } else if host.starts_with("http://") || host.starts_with("https://") { let base = host.trim_end_matches('/').to_string(); - let rpc = VmmClient::new(PrpcClient::new(format!("{base}/prpc"))); - Ok(Self { rpc, base }) + let client = PrpcClient::new(format!("{base}/prpc")); + let rpc = VmmClient::new(with_token(client)); + Ok(Self { + rpc, + base, + auth_token: token.map(str::to_string), + }) } else { bail!( "unsupported host '{host}': expected unix:/path/to/vmm.sock or http(s)://host:port" @@ -112,7 +133,14 @@ impl Vmm { /// socket and over the localhost HTTP endpoint written by `dstackup install`. pub async fn logs(&self, id: &str, lines: u32) -> Result<String> { let path = format!("/logs?id={id}&follow=false&ansi=false&lines={lines}"); - let (status, body) = http_request("GET", &self.base, &path, b"").await?; + let auth_header; + let mut headers: Vec<(&str, &str)> = Vec::new(); + if let Some(token) = &self.auth_token { + auth_header = format!("Bearer {token}"); + headers.push(("Authorization", auth_header.as_str())); + } + let (status, body) = + http_request_with_headers("GET", &self.base, &path, b"", &headers).await?; if status != 200 { bail!("vmm /logs returned status {status}"); } diff --git a/dstack/crates/dstack-cli/Cargo.toml b/dstack/crates/dstack-cli/Cargo.toml index 1ff3c41ba..d1ef71268 100644 --- a/dstack/crates/dstack-cli/Cargo.toml +++ b/dstack/crates/dstack-cli/Cargo.toml @@ -18,5 +18,11 @@ path = "src/main.rs" anyhow.workspace = true clap.workspace = true dstack-cli-core.workspace = true +dstack-volume.workspace = true +dstack-types.workspace = true +fs-err.workspace = true +hex.workspace = true serde_json.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tracing.workspace = true +tracing-subscriber = { workspace = true } diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index 1d31897d3..8a70141e9 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -11,10 +11,11 @@ //! `logs`, a global `-j/--json`). use anyhow::{bail, Context, Result}; -use clap::{Parser, Subcommand}; +use clap::{Parser, Subcommand, ValueEnum}; use dstack_cli_core::layout::InstallLayout; use dstack_cli_core::vmm::{Vmm, DEFAULT_HOST}; use dstack_cli_core::{compose, ports, rpc}; +use fs_err as fs; #[derive(Parser)] #[command( @@ -32,7 +33,9 @@ struct Cli { #[arg(long, global = true, value_name = "DIR")] prefix: Option<String>, - /// auth token for a remote VMM. + /// auth token for a VMM with `[auth]` enabled (sent as `Authorization: + /// Bearer`). Falls back to `DSTACK_VMM_TOKEN`, then the token written by + /// `dstackup install`. #[arg(long, global = true)] token: Option<String>, @@ -44,6 +47,37 @@ struct Cli { command: Command, } +#[derive(Clone, Copy, Debug, Default, ValueEnum)] +enum ComposeRunner { + #[default] + DockerCompose, + NerdctlCompose, +} + +impl ComposeRunner { + fn as_str(self) -> &'static str { + match self { + Self::DockerCompose => "docker-compose", + Self::NerdctlCompose => "nerdctl-compose", + } + } +} + +#[derive(Clone, Copy, Debug, ValueEnum)] +enum Snapshotter { + Overlayfs, + Stargz, +} + +impl Snapshotter { + fn as_str(self) -> &'static str { + match self { + Self::Overlayfs => "overlayfs", + Self::Stargz => "stargz", + } + } +} + #[derive(Subcommand)] enum Command { /// Deploy an app from a docker-compose file. @@ -68,10 +102,16 @@ enum Command { /// disk size in GB. #[arg(long, default_value_t = 20)] disk: u32, - /// expose a port: `vm` | `host:vm` | `proto:host:vm` | `proto:addr:host:vm` + /// expose a port: `vm` | `host:vm` | `proto:host:vm` | `proto:addr:host:vm`, + /// each optionally suffixed `@<nic>` to name the NIC it enters through /// (host omitted/`auto`/`0` ⇒ a free host port is picked). Repeatable. #[arg(long = "port", value_name = "SPEC")] ports: Vec<String>, + /// attach a verity volume, as printed by `dstack verity`: the file name + /// in the vmm's volumes_dir, its verity_root, and the target + /// (an absolute mount path). Repeatable. + #[arg(long = "volume", value_name = "NAME:VERITY_ROOT:TARGET")] + volumes: Vec<String>, /// deploy in non-KMS mode (ephemeral keys; no KMS required). #[arg(long)] no_kms: bool, @@ -82,6 +122,12 @@ enum Command { /// build + hash the compose and print it, without deploying. #[arg(long)] dry_run: bool, + /// compose frontend used inside the guest. + #[arg(long, value_enum, default_value = "docker-compose")] + runner: ComposeRunner, + /// containerd snapshotter (supported only with --runner nerdctl-compose). + #[arg(long, value_enum)] + snapshotter: Option<Snapshotter>, }, /// List deployed apps. Apps, @@ -100,13 +146,60 @@ enum Command { }, /// Scaffold a new app project in the current directory. Init, + /// Build a read-only verity data volume from a directory or filesystem image. + /// + /// The build needs no daemon or TEE. It prints a verity_root to paste into + /// the deploy command. See docs/verity-volumes.md. + Verity { + /// Pack this directory into a read-only data volume. + #[arg(long, value_name = "PATH")] + dir: Option<String>, + /// wrap an existing filesystem image instead of building squashfs. The + /// guest mounts it read-only after dm-verity verification. + #[arg(long = "fs-image", value_name = "PATH", conflicts_with = "dir")] + fs_image: Option<String>, + /// where to write the volume. + #[arg(long, short = 'o', default_value = "verity.img")] + output: String, + /// squashfs compression: `none` (the default), `zstd`, or `gzip`. + #[arg(long, value_enum, default_value_t, conflicts_with = "fs_image")] + compress: CompressionArg, + }, +} + +#[derive(Clone, Copy, Default, ValueEnum)] +enum CompressionArg { + #[default] + None, + Zstd, + Gzip, +} + +impl From<CompressionArg> for dstack_volume::Compression { + fn from(value: CompressionArg) -> Self { + match value { + CompressionArg::None => Self::None, + CompressionArg::Zstd => Self::Zstd, + CompressionArg::Gzip => Self::Gzip, + } + } } #[tokio::main] async fn main() -> Result<()> { + // progress (e.g. `verity` pulling layers) goes to stderr so it never mixes + // with `--json` on stdout. RUST_LOG overrides. + use tracing_subscriber::EnvFilter; + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_target(false) + .without_time() + .with_env_filter( + EnvFilter::try_from_default_env() + .unwrap_or_else(|_| EnvFilter::new("warn,dstack_volume=info")), + ) + .init(); let cli = Cli::parse(); - // remote-auth wiring lands with the TLS+token transport. - let _ = &cli.token; let defaults = LocalDefaults::read(cli.prefix.as_deref()); let use_local_defaults = cli.host.is_none(); let host = cli @@ -114,11 +207,24 @@ async fn main() -> Result<()> { .clone() .or_else(|| defaults.as_ref().and_then(|d| d.client_url.clone())) .unwrap_or_else(|| DEFAULT_HOST.to_string()); + // auth token: --token, then DSTACK_VMM_TOKEN, then the token file written + // by `dstackup install` (local defaults only). + let token = cli + .token + .clone() + .or_else(|| std::env::var("DSTACK_VMM_TOKEN").ok()) + .filter(|t| !t.trim().is_empty()) + .or_else(|| { + use_local_defaults + .then(|| defaults.as_ref().and_then(LocalDefaults::token)) + .flatten() + }); + let token = token.as_deref(); let json = cli.json; match cli.command { - Command::Apps => cmd_apps(&host, json).await, - Command::Logs { id, lines } => cmd_logs(&host, &id, lines).await, + Command::Apps => cmd_apps(&host, token, json).await, + Command::Logs { id, lines } => cmd_logs(&host, token, &id, lines).await, Command::Deploy { compose, compose_file, @@ -128,9 +234,12 @@ async fn main() -> Result<()> { memory, disk, ports, + volumes, no_kms, allowlist, dry_run, + runner, + snapshotter, } => { let compose = resolve_compose_arg(compose, compose_file)?; let image = if use_local_defaults { @@ -149,6 +258,7 @@ async fn main() -> Result<()> { }; cmd_deploy( &host, + token, &compose, &name, image.as_deref(), @@ -156,16 +266,111 @@ async fn main() -> Result<()> { memory, disk, &ports, + &volumes, no_kms, allowlist.as_deref(), dry_run, json, + runner, + snapshotter, ) .await } Command::Info { .. } => stub("info"), Command::Init => stub("init"), + Command::Verity { + dir, + fs_image, + output, + compress, + } => cmd_verity(dir.as_deref(), fs_image.as_deref(), &output, compress, json).await, + } +} + +async fn cmd_verity( + dir: Option<&str>, + fs_image: Option<&str>, + output: &str, + compress: CompressionArg, + json: bool, +) -> Result<()> { + let result = dstack_volume::verity(dstack_volume::VerityOptions { + dir: dir.map(std::path::PathBuf::from), + fs_image: fs_image.map(std::path::PathBuf::from), + output: output.into(), + compress: compress.into(), + }) + .await?; + + let volume_size = fs::metadata(&result.output) + .with_context(|| format!("stat {}", result.output.display()))? + .len(); + + if json { + print_json(&serde_json::json!({ + "verityRoot": result.verity_root, + "output": result.output.display().to_string(), + "dataSize": result.data_size, + "volumeSize": volume_size, + })); + return Ok(()); } + + let mib = volume_size as f64 / 1_048_576.0; + println!("wrote {} ({mib:.1} MiB)", result.output.display()); + let file = result + .output + .file_name() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| result.output.display().to_string()); + // a data volume mounts at a path you choose; it must be writable (the guest + // rootfs is read-only), e.g. under /run. + let target = "/run/models"; + println!("\ncopy {file} into the vmm's volumes_dir, then deploy with:"); + println!( + " dstack deploy -c docker-compose.yaml --volume {file}:{}:{target}", + result.verity_root + ); + println!(" (change {target} to your mount path)"); + Ok(()) +} + +/// Parse a `--volume` spec `NAME:VERITY_ROOT:TARGET`. +/// +/// `NAME` is the volume file in the vmm's volumes_dir. `VERITY_ROOT` and `TARGET` +/// become a measured `verity_volumes` entry in the app-compose, so the guest only +/// seeds content matching the attested root. `dstack verity` prints the exact +/// spec to paste. +/// +/// `TARGET` is an absolute read-only mount path in the guest. +fn parse_volume(spec: &str) -> Result<dstack_types::VerityVolume> { + let mut parts = spec.splitn(3, ':'); + let name = parts.next().unwrap_or_default(); + let (root, target) = match (parts.next(), parts.next()) { + (Some(root), Some(target)) if !root.is_empty() && !target.is_empty() => (root, target), + _ => bail!("--volume must be NAME:VERITY_ROOT:TARGET (as printed by `dstack verity`), got '{spec}'"), + }; + if name.is_empty() + || name.contains('/') + || name.contains("..") + || name.contains(',') + || name.contains('=') + { + bail!("volume name '{name}' must be a bare file name (no '/', '..', ',', '=')"); + } + if root.len() != 64 || !root.bytes().all(|b| b.is_ascii_hexdigit()) { + bail!("verity_root '{root}' must be 64 hex chars (copy it from `dstack verity`)"); + } + if !target.starts_with('/') { + bail!("target '{target}' must be an absolute path"); + } + let mut verity_root = [0; 32]; + hex::decode_to_slice(root, &mut verity_root).context("decoding verity_root")?; + Ok(dstack_types::VerityVolume { + source: name.to_string(), + verity_root, + target: target.into(), + }) } fn resolve_compose_arg(positional: Option<String>, flagged: Option<String>) -> Result<String> { @@ -178,6 +383,7 @@ fn resolve_compose_arg(positional: Option<String>, flagged: Option<String>) -> R struct LocalDefaults { client_url: Option<String>, + client_token_path: Option<String>, image: Option<String>, allowlist_path: Option<String>, } @@ -185,7 +391,7 @@ struct LocalDefaults { impl LocalDefaults { fn read(prefix: Option<&str>) -> Option<Self> { let path = InstallLayout::state_path_for_prefix(prefix); - let body = std::fs::read_to_string(path).ok()?; + let body = fs::read_to_string(path).ok()?; let v: serde_json::Value = serde_json::from_str(&body).ok()?; Some(Self::from_value(&v)) } @@ -197,6 +403,11 @@ impl LocalDefaults { .and_then(|x| x.as_str()) .filter(|s| !s.is_empty()) .map(str::to_string), + client_token_path: v + .get("client_token_path") + .and_then(|x| x.as_str()) + .filter(|s| !s.is_empty()) + .map(str::to_string), image: v .get("image") .and_then(|x| x.as_str()) @@ -213,100 +424,20 @@ impl LocalDefaults { fn allowlist_path(&self) -> Option<String> { self.allowlist_path.clone() } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_local_install_defaults() { - let value = serde_json::json!({ - "client_url": "http://127.0.0.1:19080", - "image": "dstack-0.5.11", - "allowlist_path": "/tmp/dstack/etc/dstack/auth-allowlist.json" - }); - let defaults = LocalDefaults::from_value(&value); - assert_eq!( - defaults.client_url.as_deref(), - Some("http://127.0.0.1:19080") - ); - assert_eq!(defaults.image.as_deref(), Some("dstack-0.5.11")); - assert_eq!( - defaults.allowlist_path().as_deref(), - Some("/tmp/dstack/etc/dstack/auth-allowlist.json") - ); - } - - #[test] - fn reads_local_install_defaults_from_prefix() { - let install_root = - std::env::temp_dir().join(format!("dstack-cli-state-test-{}", std::process::id())); - let state_dir = install_root.join("var/lib/dstack"); - std::fs::create_dir_all(&state_dir).unwrap(); - std::fs::write( - state_dir.join(dstack_cli_core::layout::STATE_FILE), - r#"{ - "client_url": "http://127.0.0.1:29080", - "image": "dstack-0.5.12", - "allowlist_path": "/tmp/custom-dstack/etc/dstack/auth-allowlist.json" - }"#, - ) - .unwrap(); - - let prefix = dstack_cli_core::layout::path_string(&install_root); - let defaults = LocalDefaults::read(Some(&prefix)).unwrap(); - assert_eq!( - defaults.client_url.as_deref(), - Some("http://127.0.0.1:29080") - ); - assert_eq!(defaults.image.as_deref(), Some("dstack-0.5.12")); - assert_eq!( - defaults.allowlist_path().as_deref(), - Some("/tmp/custom-dstack/etc/dstack/auth-allowlist.json") - ); - - let _ = std::fs::remove_dir_all(install_root); - } - #[test] - fn parses_phala_style_deploy_flags() { - let cli = Cli::parse_from([ - "dstack", - "deploy", - "-n", - "hello", - "-c", - "examples/hello-nginx/docker-compose.yaml", - "--port", - "8080:80", - ]); - match cli.command { - Command::Deploy { - compose, - compose_file, - name, - memory, - ports, - .. - } => { - assert_eq!(compose, None); - assert_eq!( - compose_file.as_deref(), - Some("examples/hello-nginx/docker-compose.yaml") - ); - assert_eq!(name, "hello"); - assert_eq!(memory, 2048); - assert_eq!(ports, vec!["8080:80"]); - } - _ => panic!("expected deploy command"), - } + /// read the VMM API token from the file recorded by `dstackup install`. + fn token(&self) -> Option<String> { + let path = self.client_token_path.as_ref()?; + let token = std::fs::read_to_string(path).ok()?; + let token = token.trim(); + (!token.is_empty()).then(|| token.to_string()) } } #[allow(clippy::too_many_arguments)] async fn cmd_deploy( host: &str, + token: Option<&str>, compose_path: &str, name: &str, image: Option<&str>, @@ -314,19 +445,40 @@ async fn cmd_deploy( memory: u32, disk: u32, port_specs: &[String], + volume_specs: &[String], no_kms: bool, allowlist: Option<&str>, dry_run: bool, json: bool, + runner: ComposeRunner, + snapshotter: Option<Snapshotter>, ) -> Result<()> { - let yaml = std::fs::read_to_string(compose_path) + if matches!(runner, ComposeRunner::DockerCompose) && snapshotter.is_some() { + bail!("--snapshotter is only supported with --runner nerdctl-compose"); + } + let yaml = fs::read_to_string(compose_path) .with_context(|| format!("reading compose file '{compose_path}'"))?; - let app_compose = compose::build_app_compose(name, &yaml, !no_kms); - let mut port_maps = Vec::new(); - for spec in port_specs { - port_maps.push(ports::parse_port(spec)?); - } + let port_maps = port_specs + .iter() + .map(|s| ports::parse_port(s)) + .collect::<Result<Vec<_>>>()?; + let parsed_volumes = volume_specs + .iter() + .map(|s| parse_volume(s)) + .collect::<Result<Vec<_>>>()?; + dstack_types::validate_verity_volumes(&parsed_volumes).map_err(anyhow::Error::msg)?; + + // each --volume declares a measured verity_volumes entry, so the built + // app-compose (and thus app_id) binds the attested roots. + let app_compose = compose::build_app_compose_with_runtime_and_volumes( + name, + &yaml, + !no_kms, + runner.as_str(), + snapshotter.map(Snapshotter::as_str), + &parsed_volumes, + ); let mut cfg = rpc::VmConfiguration { name: name.to_string(), @@ -339,7 +491,7 @@ async fn cmd_deploy( ..Default::default() }; - let vmm = Vmm::connect(host)?; + let vmm = Vmm::connect_with_token(host, token)?; let hash = vmm.get_compose_hash(&cfg).await?; let app_id = short(&hash, 40); cfg.app_id = Some(app_id.clone()); @@ -429,8 +581,8 @@ fn stub(name: &str) -> Result<()> { ) } -async fn cmd_apps(host: &str, json: bool) -> Result<()> { - let vmm = Vmm::connect(host)?; +async fn cmd_apps(host: &str, token: Option<&str>, json: bool) -> Result<()> { + let vmm = Vmm::connect_with_token(host, token)?; let resp = vmm.status().await?; if json { let arr: Vec<_> = resp @@ -470,8 +622,8 @@ async fn cmd_apps(host: &str, json: bool) -> Result<()> { Ok(()) } -async fn cmd_logs(host: &str, id: &str, lines: u32) -> Result<()> { - let vmm = Vmm::connect(host)?; +async fn cmd_logs(host: &str, token: Option<&str>, id: &str, lines: u32) -> Result<()> { + let vmm = Vmm::connect_with_token(host, token)?; let logs = vmm.logs(id, lines).await?; print!("{logs}"); Ok(()) @@ -506,3 +658,148 @@ fn trunc(s: &str, n: usize) -> String { out } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_local_install_defaults() { + let value = serde_json::json!({ + "client_url": "http://127.0.0.1:19080", + "image": "dstack-0.5.11", + "allowlist_path": "/tmp/dstack/etc/dstack/auth-allowlist.json" + }); + let defaults = LocalDefaults::from_value(&value); + assert_eq!( + defaults.client_url.as_deref(), + Some("http://127.0.0.1:19080") + ); + assert_eq!(defaults.image.as_deref(), Some("dstack-0.5.11")); + assert_eq!( + defaults.allowlist_path().as_deref(), + Some("/tmp/dstack/etc/dstack/auth-allowlist.json") + ); + } + + #[test] + fn reads_local_install_defaults_from_prefix() { + let install_root = + std::env::temp_dir().join(format!("dstack-cli-state-test-{}", std::process::id())); + let state_dir = install_root.join("var/lib/dstack"); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join(dstack_cli_core::layout::STATE_FILE), + r#"{ + "client_url": "http://127.0.0.1:29080", + "image": "dstack-0.5.12", + "allowlist_path": "/tmp/custom-dstack/etc/dstack/auth-allowlist.json" + }"#, + ) + .unwrap(); + + let prefix = dstack_cli_core::layout::path_string(&install_root); + let defaults = LocalDefaults::read(Some(&prefix)).unwrap(); + assert_eq!( + defaults.client_url.as_deref(), + Some("http://127.0.0.1:29080") + ); + assert_eq!(defaults.image.as_deref(), Some("dstack-0.5.12")); + assert_eq!( + defaults.allowlist_path().as_deref(), + Some("/tmp/custom-dstack/etc/dstack/auth-allowlist.json") + ); + + let _ = fs::remove_dir_all(install_root); + } + + #[test] + fn parses_volume_specs() { + let root = "a".repeat(64); + let data = parse_volume(&format!("weights.img:{root}:/models/llama")).unwrap(); + assert_eq!(data.source, "weights.img"); + assert_eq!(data.verity_root, [0xaa; 32]); + assert_eq!(data.target, std::path::Path::new("/models/llama")); + + assert!(parse_volume("weights.img").is_err()); // missing verity_root:target + assert!(parse_volume(&format!("weights.img:{root}")).is_err()); // missing target + assert!(parse_volume(&format!("../escape.img:{root}:/models")).is_err()); // path separator + assert!(parse_volume("x.img:nothex:/models").is_err()); // verity_root not hex + assert!(parse_volume(&format!("x.img:{root}:docker")).is_err()); // docker seed removed + assert!(parse_volume(&format!("x.img:{root}:relative/path")).is_err()); // bad target + } + + #[test] + fn parses_phala_style_deploy_flags() { + let cli = Cli::parse_from([ + "dstack", + "deploy", + "-n", + "hello", + "-c", + "examples/hello-nginx/docker-compose.yaml", + "--port", + "8080:80", + ]); + match cli.command { + Command::Deploy { + compose, + compose_file, + name, + memory, + ports, + .. + } => { + assert_eq!(compose, None); + assert_eq!( + compose_file.as_deref(), + Some("examples/hello-nginx/docker-compose.yaml") + ); + assert_eq!(name, "hello"); + assert_eq!(memory, 2048); + assert_eq!(ports, vec!["8080:80"]); + } + _ => panic!("expected deploy command"), + } + } + + #[test] + fn parses_verity_fs_image_flag() { + let cli = Cli::parse_from(["dstack", "verity", "--fs-image", "rootfs.ext4"]); + match cli.command { + Command::Verity { dir, fs_image, .. } => { + assert_eq!(dir, None); + assert_eq!(fs_image.as_deref(), Some("rootfs.ext4")); + } + _ => panic!("expected verity command"), + } + + assert!(Cli::try_parse_from([ + "dstack", + "verity", + "--dir", + "data", + "--fs-image", + "rootfs.ext4" + ]) + .is_err()); + assert!(Cli::try_parse_from([ + "dstack", + "verity", + "--fs-image", + "rootfs.ext4", + "--compress", + "zstd" + ]) + .is_err()); + assert!(Cli::try_parse_from([ + "dstack", + "verity", + "--dir", + "data", + "--compress", + "invalid" + ]) + .is_err()); + } +} diff --git a/dstack/crates/dstack-volume/Cargo.toml b/dstack/crates/dstack-volume/Cargo.toml new file mode 100644 index 000000000..76262b6fe --- /dev/null +++ b/dstack/crates/dstack-volume/Cargo.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "dstack-volume" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +anyhow = { workspace = true, features = ["std"] } +binrw.workspace = true +cmd_lib.workspace = true +clap.workspace = true +dstack-types.workspace = true +tokio = { workspace = true, features = ["rt"] } +serde_json = { workspace = true, features = ["std"] } +sha2 = { workspace = true, features = ["std"] } +hex = { workspace = true, features = ["std"] } +fs-err.workspace = true +gpt = "4.1.0" +tempfile.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +uuid.workspace = true + +[dev-dependencies] diff --git a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs new file mode 100644 index 000000000..367fc33fc --- /dev/null +++ b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs @@ -0,0 +1,523 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Discover and activate volumes supplied to a dstack guest. +//! +//! A volume is recognized by a `DSTACK_VOLUME` envelope at the start of its +//! first partition, or at the start of the whole disk when it has no partition +//! table. Everything read from a disk is untrusted: kind handlers use the +//! measured app compose as their source of policy and cryptographic identity. + +use std::collections::HashMap; +use std::io::Read; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Context, Result}; +use clap::{Parser, Subcommand}; +use cmd_lib::{run_cmd, run_fun}; +use dstack_types::{AppCompose, VerityVolume as RequestedVolume}; +use dstack_volume::volume_format::{ + DstackVolumeHeader, DSTACK_VOLUME_HEADER_SIZE, DSTACK_VOLUME_KIND_VERITY, DSTACK_VOLUME_MAGIC, +}; +use fs_err::{self as fs, File}; +use tracing::{info, warn}; + +const MAX_DISKS: usize = 64; + +#[derive(Clone, Debug, PartialEq, Eq)] +struct BlockDisk { + path: PathBuf, + partitions: Vec<(u32, PathBuf)>, +} + +#[derive(Debug)] +struct VerityVolume { + data: PathBuf, + hash: PathBuf, + root_hash: [u8; 32], +} + +#[derive(Parser)] +#[command(about = "Discover and activate dstack verity volumes")] +struct Cli { + #[command(subcommand)] + command: VolumeCommand, +} + +#[derive(Subcommand)] +enum VolumeCommand { + /// Activate every verity volume required by the measured app compose. + MountAll { + #[arg(default_value = "app-compose.json")] + compose: PathBuf, + }, + /// List recognized dstack volume devices. + Scan, + /// Compare required volumes with attached and active devices. + Status { + #[arg(default_value = "app-compose.json")] + compose: PathBuf, + }, +} + +fn main() -> Result<()> { + tracing_subscriber::fmt().init(); + match Cli::parse().command { + VolumeCommand::MountAll { compose } => mount_all(compose), + VolumeCommand::Scan => scan(), + VolumeCommand::Status { compose } => status(compose), + } +} + +fn read_compose(compose_path: &Path) -> Result<AppCompose> { + // Deserialize the complete shared type before probing any untrusted disk. + // This validates roots and targets as part of parsing the measured compose. + serde_json::from_slice(&fs::read(compose_path)?) + .with_context(|| format!("parsing {}", compose_path.display())) +} + +fn prepare_volumes() -> Result<Vec<VerityVolume>> { + let _ = run_cmd!(modprobe dm-verity); + let _ = run_cmd!(udevadm settle --timeout=5); + discover_volumes() +} + +fn mount_all(compose_path: PathBuf) -> Result<()> { + let compose = read_compose(&compose_path)?; + dstack_types::validate_verity_volumes(&compose.verity_volumes).map_err(anyhow::Error::msg)?; + if compose.verity_volumes.is_empty() { + return Ok(()); + } + let volumes = prepare_volumes()?; + info!( + found = volumes.len(), + requested = compose.verity_volumes.len(), + "discovered dstack volumes" + ); + let mut active_roots: HashMap<[u8; 32], PathBuf> = HashMap::new(); + for (index, requested) in compose.verity_volumes.iter().enumerate() { + if let Some(mapped) = active_roots.get(&requested.verity_root) { + mount_volume(requested, mapped).with_context(|| { + format!( + "failed to mount required volume {index} at {}", + requested.target.display() + ) + })?; + continue; + } + let mapped = activate_requested(index, requested, &volumes).with_context(|| { + format!( + "failed to activate required volume {index} at {}", + requested.target.display() + ) + })?; + active_roots.insert(requested.verity_root, mapped); + } + Ok(()) +} + +fn scan() -> Result<()> { + for volume in prepare_volumes()? { + println!( + "{}\tdata={}\thash={}", + hex::encode(volume.root_hash), + volume.data.display(), + volume.hash.display() + ); + } + Ok(()) +} + +fn status(compose_path: PathBuf) -> Result<()> { + let compose = read_compose(&compose_path)?; + let volumes = prepare_volumes()?; + for (index, requested) in compose.verity_volumes.iter().enumerate() { + let attached = volumes + .iter() + .any(|volume| volume.root_hash == requested.verity_root); + let expected_root = hex::encode(requested.verity_root); + let mut active = false; + for mapper_index in 0..compose.verity_volumes.len() { + let mapper_name = format!("dstack-verity{mapper_index}"); + if Path::new("/dev/mapper").join(&mapper_name).exists() + && mapping_root(&mapper_name)?.eq_ignore_ascii_case(&expected_root) + { + active = true; + break; + } + } + println!( + "{index}\troot={}\ttarget={}\tattached={attached}\tactive={active}", + hex::encode(requested.verity_root), + requested.target.display() + ); + } + Ok(()) +} + +fn discover_volumes() -> Result<Vec<VerityVolume>> { + let mut found = Vec::new(); + let disks = list_disks()?; + if disks.len() > MAX_DISKS { + warn!( + found = disks.len(), + limit = MAX_DISKS, + "block-device scan truncated" + ); + } + for disk in disks.into_iter().take(MAX_DISKS) { + // A partitioned disk has exactly one envelope location: partition 1. + // Only a disk with no kernel-recognized partitions is probed at offset 0. + let probe = if disk.partitions.is_empty() { + &disk.path + } else if let Some((_, path)) = disk.partitions.iter().find(|(number, _)| *number == 1) { + path + } else { + continue; + }; + let Some(header) = read_header(probe)? else { + continue; + }; + match header.kind { + DSTACK_VOLUME_KIND_VERITY => match resolve_verity(&disk, header) { + Ok(volume) => found.push(volume), + Err(err) => { + warn!(disk = %disk.path.display(), error = %format_args!("{err:#}"), "ignoring malformed verity volume") + } + }, + kind => warn!(kind, disk = %disk.path.display(), "ignoring unsupported volume kind"), + } + } + Ok(found) +} + +fn list_disks() -> Result<Vec<BlockDisk>> { + list_disks_at(Path::new("/sys/class/block"), Path::new("/dev")) +} + +fn list_disks_at(sysfs: &Path, devfs: &Path) -> Result<Vec<BlockDisk>> { + struct Entry { + name: std::ffi::OsString, + sysfs_path: PathBuf, + partition: Option<u32>, + } + + let mut entries = Vec::new(); + for entry in fs::read_dir(sysfs)? { + let entry = entry?; + let class_path = entry.path(); + let partition = if class_path.join("partition").exists() { + Some( + fs::read_to_string(class_path.join("partition"))? + .trim() + .parse()?, + ) + } else { + None + }; + entries.push(Entry { + name: entry.file_name(), + sysfs_path: fs::canonicalize(class_path)?, + partition, + }); + } + + let mut disks = entries + .iter() + .filter(|entry| entry.partition.is_none() && entry.sysfs_path.join("device").exists()) + .map(|entry| BlockDisk { + path: devfs.join(&entry.name), + partitions: entries + .iter() + .filter_map(|partition| { + let number = partition.partition?; + (partition.sysfs_path.parent() == Some(entry.sysfs_path.as_path())) + .then(|| (number, devfs.join(&partition.name))) + }) + .collect(), + }) + .collect::<Vec<_>>(); + for disk in &mut disks { + disk.partitions.sort_by_key(|(number, _)| *number); + } + disks.sort_by(|left, right| left.path.cmp(&right.path)); + Ok(disks) +} + +fn read_header(path: &Path) -> Result<Option<DstackVolumeHeader>> { + let mut file = match File::open(path) { + Ok(file) => file, + Err(err) => { + warn!(path = %path.display(), %err, "cannot open block device"); + return Ok(None); + } + }; + let mut bytes = [0u8; DSTACK_VOLUME_HEADER_SIZE]; + if let Err(err) = file.read_exact(&mut bytes) { + if err.kind() == std::io::ErrorKind::UnexpectedEof { + return Ok(None); + } + return Err(err).with_context(|| format!("reading {}", path.display())); + } + parse_header(&bytes).with_context(|| format!("parsing envelope on {}", path.display())) +} + +fn parse_header(bytes: &[u8]) -> Result<Option<DstackVolumeHeader>> { + if bytes.len() < DSTACK_VOLUME_HEADER_SIZE || &bytes[..16] != DSTACK_VOLUME_MAGIC { + return Ok(None); + } + Ok(Some(DstackVolumeHeader::decode(bytes)?)) +} + +fn resolve_verity(disk: &BlockDisk, header: DstackVolumeHeader) -> Result<VerityVolume> { + if disk.partitions.is_empty() { + bail!("raw verity layout is not defined by kind version 1"); + } + let partition = |number| { + disk.partitions + .iter() + .find(|(n, _)| *n == number) + .map(|(_, path)| path.clone()) + .with_context(|| format!("missing partition {number}")) + }; + Ok(VerityVolume { + data: partition(2)?, + hash: partition(3)?, + root_hash: header.root_hash, + }) +} + +fn activate_requested( + index: usize, + requested: &RequestedVolume, + volumes: &[VerityVolume], +) -> Result<PathBuf> { + let candidate = volumes + .iter() + .find(|volume| volume.root_hash == requested.verity_root) + .context("no attached volume advertises the measured root")?; + + let mapper_name = format!("dstack-verity{index}"); + let mapped = PathBuf::from(format!("/dev/mapper/{mapper_name}")); + let expected_root = hex::encode(requested.verity_root); + if mapped.exists() { + if mapping_root(&mapper_name)?.eq_ignore_ascii_case(&expected_root) { + if let Err(err) = + verify_first_block(&mapped).and_then(|_| mount_volume(requested, &mapped)) + { + let _ = run_cmd!(veritysetup close $mapper_name); + return Err(err); + } + info!(mapper = mapper_name, "reused active verity mapping"); + return Ok(mapped); + } + run_cmd!(veritysetup close $mapper_name).context("closing stale verity mapping")?; + } + + // The on-disk root only selected a candidate. Pass the root from the + // measured compose to veritysetup, which is the actual trust decision. + let data = &candidate.data; + let hash = &candidate.hash; + run_cmd!(veritysetup open $data $mapper_name $hash $expected_root) + .context("opening dm-verity volume")?; + if let Err(err) = verify_first_block(&mapped).and_then(|_| mount_volume(requested, &mapped)) { + let _ = run_cmd!(veritysetup close $mapper_name); + return Err(err); + } + Ok(mapped) +} + +fn verify_first_block(path: &Path) -> Result<()> { + let mut file = File::open(path).with_context(|| format!("opening {}", path.display()))?; + let mut block = [0u8; 4096]; + file.read_exact(&mut block) + .with_context(|| format!("verifying first block of {}", path.display())) +} + +fn mount_volume(requested: &RequestedVolume, mapped: &Path) -> Result<()> { + let fs_type = run_fun!(blkid -o value -s TYPE $mapped).unwrap_or_default(); + let target = &requested.target; + fs::create_dir_all(target)?; + if is_mountpoint(target)? { + ensure_mounted_from(target, mapped)?; + } else { + mount_read_only(mapped, target, fs_type.trim())?; + } + info!(root = %hex::encode(requested.verity_root), target = %target.display(), "mounted verity volume"); + Ok(()) +} + +fn mount_read_only(device: &Path, target: &Path, fs_type: &str) -> Result<()> { + let options = if matches!(fs_type, "ext3" | "ext4") { + "ro,noload" + } else { + "ro" + }; + if fs_type.is_empty() { + run_cmd!(mount -o $options $device $target).context("mounting verity volume")?; + } else { + run_cmd!(mount -t $fs_type -o $options $device $target) + .context("mounting verity volume")?; + } + Ok(()) +} + +fn is_mountpoint(path: &Path) -> Result<bool> { + Ok(mountpoint_device(path)?.is_some()) +} + +fn ensure_mounted_from(target: &Path, device: &Path) -> Result<()> { + let mounted = mountpoint_device(target)?.context("mount point disappeared")?; + let expected = device_number(fs::metadata(device)?.rdev()); + if mounted != expected { + bail!( + "{} is mounted from device {}:{}, expected {}:{}", + target.display(), + mounted.0, + mounted.1, + expected.0, + expected.1 + ); + } + Ok(()) +} + +fn mountpoint_device(path: &Path) -> Result<Option<(u64, u64)>> { + let target = path.as_os_str().as_bytes(); + for line in fs::read_to_string("/proc/self/mountinfo")?.lines() { + let mut fields = line.split_ascii_whitespace(); + let Some(device) = fields.nth(2) else { + continue; + }; + let Some(mountpoint) = fields.nth(1) else { + continue; + }; + if unescape_mountinfo(mountpoint.as_bytes()) == target { + let (major, minor) = device + .split_once(':') + .context("invalid mountinfo device number")?; + return Ok(Some((major.parse()?, minor.parse()?))); + } + } + Ok(None) +} + +fn device_number(device: u64) -> (u64, u64) { + let major = ((device >> 8) & 0xfff) | ((device >> 32) & 0xffff_f000); + let minor = (device & 0xff) | ((device >> 12) & 0xffff_ff00); + (major, minor) +} + +fn unescape_mountinfo(value: &[u8]) -> Vec<u8> { + let mut decoded = Vec::with_capacity(value.len()); + let mut index = 0; + while index < value.len() { + if value[index] == b'\\' && index + 3 < value.len() { + let octal = &value[index + 1..index + 4]; + if octal.iter().all(|byte| matches!(byte, b'0'..=b'7')) { + decoded.push((octal[0] - b'0') * 64 + (octal[1] - b'0') * 8 + (octal[2] - b'0')); + index += 4; + continue; + } + } + decoded.push(value[index]); + index += 1; + } + decoded +} + +fn mapping_root(mapper_name: &str) -> Result<String> { + let status = run_fun!(veritysetup status $mapper_name)?; + status + .lines() + .find_map(|line| { + let line = line.trim(); + line.strip_prefix("root hash:") + .or_else(|| line.strip_prefix("Root hash:")) + }) + .map(|root| root.trim().to_string()) + .context("verity mapping status has no root hash") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cli_uses_default_compose_for_mount_all() { + let cli = Cli::try_parse_from(["dstack-volume", "mount-all"]).unwrap(); + let VolumeCommand::MountAll { compose } = cli.command else { + panic!("unexpected command"); + }; + assert_eq!(compose, Path::new("app-compose.json")); + } + + fn header() -> DstackVolumeHeader { + DstackVolumeHeader::new_verity([0x5a; 32]) + } + + #[test] + fn ignores_non_volume_data() { + assert_eq!( + parse_header(&[0u8; DSTACK_VOLUME_HEADER_SIZE]).unwrap(), + None + ); + } + + #[test] + fn verity_kind_uses_second_and_third_partitions() { + let disk = BlockDisk { + path: "/dev/test".into(), + partitions: vec![ + (1, "/dev/test1".into()), + (2, "/dev/test2".into()), + (3, "/dev/test3".into()), + ], + }; + let volume = resolve_verity(&disk, header()).unwrap(); + assert_eq!(volume.data, Path::new("/dev/test2")); + assert_eq!(volume.hash, Path::new("/dev/test3")); + } + + #[test] + fn verity_kind_rejects_undefined_raw_layout() { + let disk = BlockDisk { + path: "/dev/test".into(), + partitions: vec![], + }; + assert!(resolve_verity(&disk, header()).is_err()); + } + + #[test] + fn decodes_mountinfo_escapes() { + assert_eq!(unescape_mountinfo(b"/run/my\\040volume"), b"/run/my volume"); + } + + #[test] + fn discovers_partitions_from_sysfs_parentage() -> Result<()> { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir()?; + let sysfs = temp.path().join("sys/class/block"); + let devices = temp.path().join("sys/devices/block/vda"); + let partition = devices.join("vda1"); + fs::create_dir_all(&sysfs)?; + fs::create_dir_all(devices.join("device"))?; + fs::create_dir_all(&partition)?; + fs::write(partition.join("partition"), "1\n")?; + symlink(&devices, sysfs.join("vda"))?; + symlink(&partition, sysfs.join("vda1"))?; + + assert_eq!( + list_disks_at(&sysfs, Path::new("/dev"))?, + vec![BlockDisk { + path: "/dev/vda".into(), + partitions: vec![(1, "/dev/vda1".into())], + }] + ); + Ok(()) + } +} diff --git a/dstack/crates/dstack-volume/src/lib.rs b/dstack/crates/dstack-volume/src/lib.rs new file mode 100644 index 000000000..5a8aa5f28 --- /dev/null +++ b/dstack/crates/dstack-volume/src/lib.rs @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Build, describe, and activate dstack volumes. +//! +//! The output is a reproducible, dm-verity-protected raw disk image containing +//! a filesystem that the guest mounts read-only at a measured path. +//! +//! The build needs no docker daemon and no TEE, and it's reproducible: the same +//! inputs always give the same `verity_root`. The first partition contains a +//! generic `DSTACK_VOLUME` envelope, followed by data and verity partitions. +//! So anyone can recompute the root +//! and check it against `app-compose.json`, without trusting the builder. See +//! docs/verity-volumes.md. + +use std::path::PathBuf; + +use anyhow::{bail, Context, Result}; + +mod volume; +pub mod volume_format; + +pub use volume::Compression; + +/// A fixed dm-verity salt. +/// +/// The root is a function of the squashfs bytes and this salt, so keeping the +/// salt constant is what lets anyone recompute the root. It isn't a secret: +/// veritysetup writes it into the on-disk verity superblock anyway. +const VERITY_SALT: &str = "0000000000000000000000000000000000000000000000000000000000000000"; + +pub struct VerityOptions { + /// Build a volume from this directory. + pub dir: Option<PathBuf>, + /// wrap an existing filesystem image as the verity data partition. This is + /// for hand-built ext4/xfs/etc. images; `dstack verity --dir` still produces + /// squashfs by default. + pub fs_image: Option<PathBuf>, + pub output: PathBuf, + /// squashfs compression (default: none — zero decompression at read time). + pub compress: Compression, +} + +pub struct VerityResult { + pub verity_root: String, + pub data_size: u64, + pub output: PathBuf, +} + +pub async fn verity(opts: VerityOptions) -> Result<VerityResult> { + match (opts.dir, opts.fs_image) { + (Some(dir), None) => verity_dir(dir, opts.output, opts.compress).await, + (None, Some(fs_image)) => verity_fs_image(fs_image, opts.output).await, + _ => bail!("give exactly one source: --dir <path> or --fs-image <path>"), + } +} + +/// Bake a directory tree into a reproducible squashfs data volume. +async fn verity_dir(dir: PathBuf, output: PathBuf, compress: Compression) -> Result<VerityResult> { + if !dir.is_dir() { + bail!("--dir '{}' is not a directory", dir.display()); + } + let out = output.clone(); + let built = tokio::task::spawn_blocking(move || { + volume::build_volume(&dir, &out, VERITY_SALT, compress) + }) + .await + .context("the build task failed")??; + + Ok(VerityResult { + verity_root: built.verity_root, + data_size: built.data_size, + output, + }) +} + +/// Wrap an already-built filesystem image. The guest discovers and mounts the +/// filesystem only after dm-verity is active. +async fn verity_fs_image(fs_image: PathBuf, output: PathBuf) -> Result<VerityResult> { + if !fs_image.is_file() { + bail!("--fs-image '{}' is not a file", fs_image.display()); + } + let out = output.clone(); + let built = + tokio::task::spawn_blocking(move || volume::build_fs_image(&fs_image, &out, VERITY_SALT)) + .await + .context("the build task failed")??; + + Ok(VerityResult { + verity_root: built.verity_root, + data_size: built.data_size, + output, + }) +} diff --git a/dstack/crates/dstack-volume/src/volume.rs b/dstack/crates/dstack-volume/src/volume.rs new file mode 100644 index 000000000..6c8175788 --- /dev/null +++ b/dstack/crates/dstack-volume/src/volume.rs @@ -0,0 +1,504 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Pack a directory into a reproducible, verity-protected disk image. +//! +//! The output is a raw disk with three deterministic GPT partitions: +//! +//! 1. a generic `DSTACK_VOLUME` metadata block +//! 2. the filesystem image +//! 3. the dm-verity superblock and hash tree +//! +//! Keeping the verity data and hash devices in separate partitions means the +//! guest never has to inspect filesystem metadata to find the hash tree. The +//! partition table is only a locator hint; the measured verity root is still the +//! content identity. + +use std::collections::BTreeMap; +use std::path::Path; + +use crate::volume_format::{DstackVolumeHeader, DSTACK_VOLUME_HEADER_SIZE}; +use anyhow::{bail, Context, Result}; +use cmd_lib::{run_cmd, run_fun}; +use fs_err as fs; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +/// Fixed build timestamp (2024-01-01), so the image is the same no matter when +/// it's built. It never shows up at runtime: `mksquashfs -all-time` normalizes +/// the store's own timestamps away. +const EPOCH: &str = "1704067200"; +const BLOCK: u64 = 4096; +const SECTOR: u64 = 512; +// 1 MiB alignment. +const PARTITION_ALIGNMENT_SECTORS: u64 = 2048; +// The `gpt` crate writes the primary/backup headers and partition arrays. We +// still reserve enough trailing sectors in the raw image for the backup array +// (128 entries * 128 bytes) plus the backup header. +const GPT_ENTRY_SECTORS: u64 = 32; +const VOLUME_HEADER_SIZE: usize = DSTACK_VOLUME_HEADER_SIZE; + +#[derive(Clone, Copy)] +pub enum Compression { + /// No compression, so nothing is decompressed at read time (the default). + None, + Zstd, + Gzip, +} + +impl Compression { + fn args(self) -> Vec<&'static str> { + match self { + // disable every compressible section: inodes, data, fragments, + // xattrs, id table. + Compression::None => vec!["-noI", "-noD", "-noF", "-noX", "-noId"], + Compression::Zstd => vec!["-comp", "zstd"], + Compression::Gzip => vec![], + } + } +} + +pub struct BuiltVolume { + pub verity_root: String, + /// Size of the filesystem data partition. + pub data_size: u64, +} + +/// Build `output`: a GPT disk image with the squashfs data partition and a +/// separate dm-verity hash partition. +/// +/// `store` is the directory to pack; `salt` fixes the verity root. +/// +/// The verity UUID is derived from the squashfs bytes, so two different volumes +/// get two different UUIDs. That matters because a VM can mount several volumes +/// at once, and a fixed UUID would be shared by all of them. +pub fn build_volume( + store: &Path, + output: &Path, + salt_hex: &str, + compress: Compression, +) -> Result<BuiltVolume> { + require_tool("mksquashfs")?; + require_tool("veritysetup")?; + + let tmp = tempfile::tempdir().context("creating volume scratch dir")?; + let data_path = tmp.path().join("data.fs"); + + // 1. reproducible squashfs. + let compression_args = compress.args(); + run_cmd!( + mksquashfs $store $data_path $[compression_args] + -all-time $EPOCH -mkfs-time $EPOCH -noappend -no-progress -xattrs + >/dev/null + ) + .context("running mksquashfs")?; + + // 2. the verity data region must be block-aligned; pad the squashfs up. + let bytes_used = squashfs_bytes_used(&data_path)?; + seal_data_image(&data_path, bytes_used, output, salt_hex) +} + +/// Wrap an existing filesystem image in the same partitioned verity disk format. +/// +/// The input image is copied to a scratch file and padded to a 4096-byte verity +/// block boundary. Its filesystem type is otherwise opaque to the verity layer. +pub fn build_fs_image(fs_image: &Path, output: &Path, salt_hex: &str) -> Result<BuiltVolume> { + require_tool("veritysetup")?; + let len = fs::metadata(fs_image) + .with_context(|| format!("stat {}", fs_image.display()))? + .len(); + if len == 0 { + bail!("filesystem image '{}' is empty", fs_image.display()); + } + + let tmp = tempfile::tempdir().context("creating volume scratch dir")?; + let data_path = tmp.path().join("data.fs"); + fs::copy(fs_image, &data_path) + .with_context(|| format!("copying {} into scratch", fs_image.display()))?; + seal_data_image(&data_path, len, output, salt_hex) +} + +fn seal_data_image( + data_path: &Path, + data_len: u64, + output: &Path, + salt_hex: &str, +) -> Result<BuiltVolume> { + let hash_tmp = tempfile::tempdir().context("creating verity hash scratch dir")?; + let hash_path = hash_tmp.path().join("verity.hash"); + let data_size = data_len.div_ceil(BLOCK) * BLOCK; + { + let f = fs::OpenOptions::new().write(true).open(data_path)?; + f.set_len(data_size)?; + } + + // Build the verity hash device as a separate image. At runtime this is + // partition 2, so no --hash-offset is needed and the guest does not parse + // filesystem metadata before verity is active. + // Pin the superblock UUID too: veritysetup randomizes it otherwise, and we + // want the whole image reproducible, not just the root. The UUID sits in the + // hash tree, not the hashed data, so it never changes the root. + let uuid = uuid_from_data(data_path, data_size)?; + let out = run_fun!( + veritysetup format --salt $salt_hex --uuid $uuid + --hash sha256 --format 1 --data-block-size 4096 --hash-block-size 4096 + $data_path $hash_path + ) + .context("running veritysetup format")?; + let verity_root = + parse_root_hash(&out).context("could not find the root hash in veritysetup output")?; + + // Wrap the two blobs in a deterministic GPT disk image. Partition 1 is the + // generic volume envelope, partition 2 is data, and partition 3 is verity. + build_partitioned_image(data_path, data_size, &hash_path, output, &verity_root)?; + + Ok(BuiltVolume { + verity_root, + data_size, + }) +} + +/// squashfs superblock: `bytes_used` is a little-endian u64 at offset 40. +fn squashfs_bytes_used(path: &Path) -> Result<u64> { + use std::io::{Read, Seek, SeekFrom}; + let mut f = fs::File::open(path)?; + let mut magic = [0u8; 4]; + f.read_exact(&mut magic)?; + if &magic != b"hsqs" { + bail!("not a squashfs image (bad magic)"); + } + f.seek(SeekFrom::Start(40))?; + let mut buf = [0u8; 8]; + f.read_exact(&mut buf)?; + Ok(u64::from_le_bytes(buf)) +} + +/// Derive a UUID from the first `len` bytes of `path`. +fn uuid_from_data(path: &Path, len: u64) -> Result<String> { + Ok(uuid_from_file(path, len)?.to_string()) +} + +/// Deterministic, version-4-shaped UUID from arbitrary domain-separated +/// material. +fn uuid_from_material(parts: &[&[u8]]) -> Uuid { + let mut h = Sha256::new(); + for part in parts { + h.update((part.len() as u64).to_le_bytes()); + h.update(part); + } + uuid_from_digest(&h.finalize()) +} + +fn uuid_from_file(path: &Path, len: u64) -> Result<Uuid> { + use std::io::Read; + let mut f = fs::File::open(path)?; + let mut h = Sha256::new(); + let mut remaining = len; + let mut buf = vec![0u8; 1 << 20]; + while remaining > 0 { + let n = remaining.min(buf.len() as u64) as usize; + f.read_exact(&mut buf[..n])?; + h.update(&buf[..n]); + remaining -= n as u64; + } + Ok(uuid_from_digest(&h.finalize())) +} + +fn uuid_from_digest(d: &[u8]) -> Uuid { + let mut u = [0u8; 16]; + u.copy_from_slice(&d[..16]); + u[6] = (u[6] & 0x0f) | 0x40; // version 4 + u[8] = (u[8] & 0x3f) | 0x80; // RFC 4122 variant + Uuid::from_bytes(u) +} + +#[derive(Clone, Copy)] +struct Partition { + first_lba: u64, + last_lba: u64, +} + +struct GptLayout { + total_lbas: u64, + metadata: Partition, + data: Partition, + hash: Partition, +} + +impl GptLayout { + fn new(data_size: u64, hash_size: u64) -> Result<Self> { + if data_size == 0 || hash_size == 0 { + bail!("data and hash images must be non-empty"); + } + let data_sectors = data_size.div_ceil(SECTOR); + let hash_sectors = hash_size.div_ceil(SECTOR); + + let metadata_first = PARTITION_ALIGNMENT_SECTORS; + let metadata_last = metadata_first + VOLUME_HEADER_SIZE as u64 / SECTOR - 1; + let data_first = align_up(metadata_last + 1, PARTITION_ALIGNMENT_SECTORS); + let data_last = data_first + data_sectors - 1; + let hash_first = align_up(data_last + 1, PARTITION_ALIGNMENT_SECTORS); + let hash_last = hash_first + hash_sectors - 1; + + // Leave room for the backup GPT entry array and header at the end. + let min_lbas = hash_last + 1 + GPT_ENTRY_SECTORS + 1; + let total_lbas = align_up(min_lbas, PARTITION_ALIGNMENT_SECTORS); + Ok(Self { + total_lbas, + metadata: Partition { + first_lba: metadata_first, + last_lba: metadata_last, + }, + data: Partition { + first_lba: data_first, + last_lba: data_last, + }, + hash: Partition { + first_lba: hash_first, + last_lba: hash_last, + }, + }) + } + + fn total_bytes(&self) -> u64 { + self.total_lbas * SECTOR + } +} + +fn align_up(value: u64, alignment: u64) -> u64 { + value.div_ceil(alignment) * alignment +} + +fn build_partitioned_image( + data_path: &Path, + data_size: u64, + hash_path: &Path, + output: &Path, + root_hash: &str, +) -> Result<()> { + let hash_size = fs::metadata(hash_path) + .with_context(|| format!("stat {}", hash_path.display()))? + .len(); + let layout = GptLayout::new(data_size, hash_size)?; + + let mut out = fs::OpenOptions::new() + .create(true) + .truncate(true) + .read(true) + .write(true) + .open(output) + .with_context(|| format!("creating {}", output.display()))?; + out.set_len(layout.total_bytes())?; + + out = write_gpt(out, &layout, root_hash)?; + + write_volume_header(&mut out, layout.metadata.first_lba * SECTOR, root_hash)?; + copy_into(data_path, &mut out, layout.data.first_lba * SECTOR)?; + copy_into(hash_path, &mut out, layout.hash.first_lba * SECTOR)?; + Ok(()) +} + +fn write_gpt(mut out: fs::File, layout: &GptLayout, root_hash: &str) -> Result<fs::File> { + let protective_size = (layout.total_lbas - 1).min(u32::MAX as u64) as u32; + gpt::mbr::ProtectiveMBR::with_lb_size(protective_size) + .overwrite_lba0(&mut out) + .context("writing protective MBR")?; + + let disk_uuid = uuid_from_material(&[b"dstack-verity-disk", root_hash.as_bytes()]); + let mut disk = gpt::GptConfig::new() + .writable(true) + .logical_block_size(gpt::disk::LogicalBlockSize::Lb512) + .create_from_device(out, Some(disk_uuid)) + .context("initializing GPT")?; + + let mut parts = BTreeMap::new(); + parts.insert( + 1, + gpt::partition::Partition { + part_type_guid: gpt::partition_types::LINUX_FS, + part_guid: uuid_from_material(&[b"dstack-volume-metadata", root_hash.as_bytes()]), + first_lba: layout.metadata.first_lba, + last_lba: layout.metadata.last_lba, + flags: 0, + name: "dstack-volume".to_string(), + }, + ); + parts.insert( + 2, + gpt::partition::Partition { + part_type_guid: gpt::partition_types::LINUX_FS, + part_guid: uuid_from_material(&[b"dstack-verity-data", root_hash.as_bytes()]), + first_lba: layout.data.first_lba, + last_lba: layout.data.last_lba, + flags: 0, + name: "dstack-data".to_string(), + }, + ); + parts.insert( + 3, + gpt::partition::Partition { + part_type_guid: gpt::partition_types::LINUX_FS, + part_guid: uuid_from_material(&[b"dstack-verity-hash", root_hash.as_bytes()]), + first_lba: layout.hash.first_lba, + last_lba: layout.hash.last_lba, + flags: 0, + name: "dstack-verity".to_string(), + }, + ); + disk.update_partitions(parts) + .context("installing GPT partitions")?; + disk.write().context("writing GPT") +} + +fn write_volume_header(out: &mut fs::File, offset: u64, root_hash: &str) -> Result<()> { + use std::io::{Seek, SeekFrom, Write}; + + let root = hex::decode(root_hash).context("decoding verity root hash")?; + let root: [u8; 32] = root + .as_slice() + .try_into() + .context("verity root must be a 32-byte SHA-256 digest")?; + let header = DstackVolumeHeader::new_verity(root) + .encode() + .context("encoding volume header")?; + + out.seek(SeekFrom::Start(offset))?; + out.write_all(&header)?; + Ok(()) +} + +fn copy_into(src: &Path, out: &mut fs::File, offset: u64) -> Result<()> { + use std::io::{Read, Seek, SeekFrom, Write}; + let mut input = fs::File::open(src)?; + out.seek(SeekFrom::Start(offset))?; + let mut buf = vec![0u8; 1 << 20]; + loop { + let n = input.read(&mut buf)?; + if n == 0 { + break; + } + out.write_all(&buf[..n])?; + } + Ok(()) +} + +fn parse_root_hash(output: &str) -> Option<String> { + output + .lines() + .find_map(|l| l.strip_prefix("Root hash:")) + .map(|v| v.trim().to_string()) +} + +fn require_tool(name: &str) -> Result<()> { + let present = run_cmd!(which $name >/dev/null 2>&1).is_ok(); + if !present { + bail!("`{name}` not found on PATH (install squashfs-tools / cryptsetup)"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_veritysetup_root() { + let sample = "VERITY header information for x\nUUID: \nHash type: 1\n\ + Data blocks: 10\nRoot hash: abc123def\n"; + assert_eq!(parse_root_hash(sample).as_deref(), Some("abc123def")); + } + + #[test] + fn uuid_is_deterministic_and_content_specific() { + use std::io::Write; + let mut a = tempfile::NamedTempFile::new().unwrap(); + a.write_all(b"hello world data").unwrap(); + let mut b = tempfile::NamedTempFile::new().unwrap(); + b.write_all(b"different data..").unwrap(); + let ua = uuid_from_data(a.path(), 16).unwrap(); + assert_eq!(ua, uuid_from_data(a.path(), 16).unwrap()); // deterministic + assert_ne!(ua, uuid_from_data(b.path(), 16).unwrap()); // content-specific + assert_eq!(ua.len(), 36); + assert_eq!(&ua[14..15], "4"); // version nibble + } + + #[test] + fn gpt_layout_uses_three_aligned_partitions() { + let layout = GptLayout::new(4096, 8192).unwrap(); + assert_eq!(layout.metadata.first_lba, PARTITION_ALIGNMENT_SECTORS); + assert_eq!(layout.metadata.last_lba, PARTITION_ALIGNMENT_SECTORS + 7); + assert_eq!(layout.data.first_lba, PARTITION_ALIGNMENT_SECTORS * 2); + assert_eq!(layout.data.last_lba, PARTITION_ALIGNMENT_SECTORS * 2 + 7); + assert_eq!(layout.hash.first_lba, PARTITION_ALIGNMENT_SECTORS * 3); + assert_eq!(layout.hash.last_lba, PARTITION_ALIGNMENT_SECTORS * 3 + 15); + assert!(layout.total_lbas - 1 - GPT_ENTRY_SECTORS > layout.hash.last_lba); + } + + #[test] + fn partitioned_image_is_valid_gpt() -> Result<()> { + use std::io::{Read, Seek, SeekFrom, Write}; + + let tmp = tempfile::tempdir()?; + let data_path = tmp.path().join("data.fs"); + let hash_path = tmp.path().join("verity.hash"); + let image_path = tmp.path().join("volume.img"); + let data = vec![0x11; 4096]; + let hash = vec![0x22; 8192]; + fs::File::create(&data_path)?.write_all(&data)?; + fs::File::create(&hash_path)?.write_all(&hash)?; + + let root_hash = "abababababababababababababababababababababababababababababababab"; + build_partitioned_image( + &data_path, + data.len() as u64, + &hash_path, + &image_path, + root_hash, + )?; + + let disk = gpt::GptConfig::new() + .logical_block_size(gpt::disk::LogicalBlockSize::Lb512) + .open(&image_path)?; + assert_eq!( + *disk.guid(), + uuid_from_material(&[b"dstack-verity-disk", root_hash.as_bytes()]) + ); + + let metadata = disk.partitions().get(&1).unwrap(); + let data_partition = disk.partitions().get(&2).unwrap(); + let hash_partition = disk.partitions().get(&3).unwrap(); + assert_eq!(metadata.name, "dstack-volume"); + assert_eq!(data_partition.name, "dstack-data"); + assert_eq!(hash_partition.name, "dstack-verity"); + assert_eq!( + data_partition.part_guid, + uuid_from_material(&[b"dstack-verity-data", root_hash.as_bytes()]) + ); + assert_eq!( + hash_partition.part_guid, + uuid_from_material(&[b"dstack-verity-hash", root_hash.as_bytes()]) + ); + assert_eq!(metadata.first_lba, PARTITION_ALIGNMENT_SECTORS); + assert_eq!(data_partition.first_lba, PARTITION_ALIGNMENT_SECTORS * 2); + assert_eq!(hash_partition.first_lba, PARTITION_ALIGNMENT_SECTORS * 3); + + let mut img = fs::File::open(&image_path)?; + let mut header = [0u8; VOLUME_HEADER_SIZE]; + img.seek(SeekFrom::Start(metadata.first_lba * SECTOR))?; + img.read_exact(&mut header)?; + assert_eq!(&header[..16], b"DSTACK_VOLUME\0\0\0"); + let decoded = DstackVolumeHeader::decode(&header)?; + assert_eq!(decoded.root_hash.as_slice(), hex::decode(root_hash)?); + let mut buf = vec![0; data.len()]; + img.seek(SeekFrom::Start(data_partition.first_lba * SECTOR))?; + img.read_exact(&mut buf)?; + assert_eq!(buf, data); + let mut buf = vec![0; hash.len()]; + img.seek(SeekFrom::Start(hash_partition.first_lba * SECTOR))?; + img.read_exact(&mut buf)?; + assert_eq!(buf, hash); + + Ok(()) + } +} diff --git a/dstack/crates/dstack-volume/src/volume_format.rs b/dstack/crates/dstack-volume/src/volume_format.rs new file mode 100644 index 000000000..df9fd4328 --- /dev/null +++ b/dstack/crates/dstack-volume/src/volume_format.rs @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! On-disk dstack volume envelope shared by builders and guests. + +use std::io::Cursor; + +use binrw::{binrw, BinRead, BinWrite}; + +pub const DSTACK_VOLUME_MAGIC: &[u8; 16] = b"DSTACK_VOLUME\0\0\0"; +pub const DSTACK_VOLUME_HEADER_SIZE: usize = 4096; +pub const DSTACK_VOLUME_KIND_VERITY: u32 = 1; + +#[binrw] +#[brw(little, magic = b"DSTACK_VOLUME\0\0\0")] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DstackVolumeHeader { + pub kind: u32, + pub root_hash: [u8; 32], +} + +impl DstackVolumeHeader { + pub fn new_verity(root_hash: [u8; 32]) -> Self { + Self { + kind: DSTACK_VOLUME_KIND_VERITY, + root_hash, + } + } + + pub fn encode(&self) -> binrw::BinResult<[u8; DSTACK_VOLUME_HEADER_SIZE]> { + let mut block = [0u8; DSTACK_VOLUME_HEADER_SIZE]; + self.write(&mut Cursor::new(&mut block[..]))?; + Ok(block) + } + + pub fn decode(block: &[u8]) -> binrw::BinResult<Self> { + if block.len() < DSTACK_VOLUME_HEADER_SIZE { + return Err(binrw::Error::AssertFail { + pos: 0, + message: format!( + "volume header is truncated: {} < {DSTACK_VOLUME_HEADER_SIZE}", + block.len() + ), + }); + } + Self::read(&mut Cursor::new(block)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use dstack_types::VerityVolume; + + #[test] + fn volume_header_round_trip() { + let expected = DstackVolumeHeader::new_verity([0x5a; 32]); + let encoded = expected.encode().unwrap(); + assert_eq!(&encoded[..16], b"DSTACK_VOLUME\0\0\0"); + assert_eq!(DstackVolumeHeader::decode(&encoded).unwrap(), expected); + } + + #[test] + fn verity_volume_validates_root_and_target_during_deserialization( + ) -> Result<(), serde_json::Error> { + let volume: VerityVolume = serde_json::from_value(serde_json::json!({ + "source": "models.img", + "verity_root": "5a".repeat(32), + "target": "/run/models" + }))?; + assert_eq!(volume.verity_root, [0x5a; 32]); + assert_eq!(volume.target, std::path::PathBuf::from("/run/models")); + assert_eq!( + serde_json::to_value(&volume)?["verity_root"], + "5a".repeat(32) + ); + + assert!(serde_json::from_value::<VerityVolume>(serde_json::json!({ + "source": "models.img", + "verity_root": "abcd", + "target": "/run/models" + })) + .is_err()); + assert!(serde_json::from_value::<VerityVolume>(serde_json::json!({ + "source": "models.img", + "verity_root": "5a".repeat(32), + "target": "relative/path" + })) + .is_err()); + Ok(()) + } +} diff --git a/dstack/crates/dstackup/Cargo.toml b/dstack/crates/dstackup/Cargo.toml index 465008c85..11bca8fc5 100644 --- a/dstack/crates/dstackup/Cargo.toml +++ b/dstack/crates/dstackup/Cargo.toml @@ -17,6 +17,7 @@ anyhow.workspace = true clap.workspace = true dstack-cli-core.workspace = true hex = { workspace = true, features = ["alloc"] } +rand.workspace = true # reqwest (+ rustls/hyper) is already linked via dstack-cli-core's prpc client, # so using it for http here adds ~no binary cost and drops the curl dependency. reqwest = { workspace = true } diff --git a/dstack/crates/dstackup/src/cid.rs b/dstack/crates/dstackup/src/cid.rs index f0cc44ae7..73291b08f 100644 --- a/dstack/crates/dstackup/src/cid.rs +++ b/dstack/crates/dstackup/src/cid.rs @@ -5,52 +5,126 @@ //! pick a vsock CID window that doesn't collide with a VMM already on the host. use anyhow::{bail, Result}; +use rand::Rng; -/// size of a VMM's CID pool (matches `config::VmmRender` default). +/// size of a VMM's CID pool (matches `config::VmmRender` default). This is what +/// ends up in `cvm.cid_pool_size`. const CID_POOL_SIZE: u32 = 1000; -/// default CID pool start when nothing else is using that range. -const DEFAULT_CID_START: u32 = 1000; -/// whether `[start, start+CID_POOL_SIZE)` intersects any occupied range. -fn cid_window_overlaps(start: u32, occupied: &[(u32, u32)]) -> bool { - let end = start.saturating_add(CID_POOL_SIZE); +/// spacing between candidate pool starts, and the width we require to be free +/// before taking one. +/// +/// This is an installer-side heuristic only: it is never written to vmm.toml and +/// the VMM knows nothing about it. Requiring a whole stride to be free leaves an +/// instance room to raise `cvm.cid_pool_size` later without walking into a +/// neighbour, and striding the candidate space keeps a random pick cheap to +/// verify. +const CID_BLOCK_STRIDE: u32 = 10_000; + +/// lowest candidate start. The first stride is skipped because CIDs 0-2 are +/// reserved (hypervisor, local, host). +const CID_MIN: u32 = CID_BLOCK_STRIDE; + +/// highest candidate start, chosen so `start + CID_BLOCK_STRIDE` stays in u32. +const CID_MAX: u32 = u32::MAX - CID_BLOCK_STRIDE; + +/// how many candidates to try before giving up. +const CID_PICK_ATTEMPTS: usize = 8; + +/// whether `[start, start + width)` intersects any occupied range. Both sides +/// are half-open. +fn window_overlaps(start: u32, width: u32, occupied: &[(u32, u32)]) -> bool { + let end = start.saturating_add(width); occupied.iter().any(|&(s, e)| start < e && s < end) } -/// the lowest pool-aligned CID block at or above every occupied range. We jump -/// above the highest reservation rather than packing into a free gap below it — -/// simpler, and the result is always collision-free. -fn next_free_cid_block(occupied: &[(u32, u32)]) -> u32 { - let max_end = occupied - .iter() - .map(|&(_, e)| e) - .max() - .unwrap_or(DEFAULT_CID_START); - (max_end.div_ceil(CID_POOL_SIZE) * CID_POOL_SIZE).max(DEFAULT_CID_START) +/// number of stride-aligned candidate blocks in `[CID_MIN, CID_MAX]`. +fn candidate_blocks() -> u32 { + (CID_MAX - CID_MIN) / CID_BLOCK_STRIDE + 1 +} + +/// an endless stream of stride-aligned starts drawn uniformly from the CID space. +/// +/// Random rather than "the next free block above everything in use": two installs +/// racing on the same host would compute the same next block from the same +/// observation and collide deterministically, and a single stray high CID from an +/// unrelated QEMU would drag every later install up with it. +fn random_starts() -> impl Iterator<Item = u32> { + let mut rng = rand::thread_rng(); + let blocks = candidate_blocks(); + std::iter::repeat_with(move || CID_MIN + rng.gen_range(0..blocks) * CID_BLOCK_STRIDE) +} + +/// first candidate whose whole stride is free, within the attempt budget. +fn first_free(candidates: impl Iterator<Item = u32>, occupied: &[(u32, u32)]) -> Option<u32> { + candidates + .take(CID_PICK_ATTEMPTS) + .find(|&start| !window_overlaps(start, CID_BLOCK_STRIDE, occupied)) +} + +/// choose a CID window that won't collide with a VMM already on this host. +/// +/// Precedence: an explicit `--cid-start` is honored but refused on overlap; then +/// the start a previous install recorded; then a random free block. +pub(crate) fn pick_cid_start( + explicit: Option<u32>, + recorded: Option<u32>, + occupied: &[(u32, u32)], +) -> Result<u32> { + pick_cid_start_from(explicit, recorded, occupied, random_starts()) } -/// choose a CID window `[start, start+CID_POOL_SIZE)` that won't collide with a -/// VMM already running on this host. With an explicit `--cid-start`, honor it -/// but refuse on overlap; without one, use the default unless it's taken, then -/// move to the next free block. -pub(crate) fn pick_cid_start(explicit: Option<u32>, occupied: &[(u32, u32)]) -> Result<u32> { - match explicit { - Some(n) => { - if cid_window_overlaps(n, occupied) { - bail!( - "--cid-start {n} overlaps a CID range already reserved on this host; \ - pick a free start, e.g. --cid-start {}", - next_free_cid_block(occupied) - ); +fn pick_cid_start_from( + explicit: Option<u32>, + recorded: Option<u32>, + occupied: &[(u32, u32)], + candidates: impl Iterator<Item = u32>, +) -> Result<u32> { + if let Some(start) = explicit { + // A start so high that its pool runs off the end of the CID space. The + // VMM rejects this too (`Config::validate`), but preflight is where it + // belongs: nothing has been written to the host yet. + if start.checked_add(CID_POOL_SIZE).is_none() { + bail!( + "--cid-start {start} leaves no room for a {CID_POOL_SIZE}-wide pool \ + (it overflows u32)" + ); + } + // An explicit choice is checked against the pool it actually asks for, + // not the stride: the operator picked the number, so don't demand + // growth headroom they didn't ask for. + if window_overlaps(start, CID_POOL_SIZE, occupied) { + match first_free(candidates, occupied) { + Some(free) => bail!( + "--cid-start {start} overlaps a CID range already reserved on this host; \ + pick a free start, e.g. --cid-start {free}" + ), + None => bail!( + "--cid-start {start} overlaps a CID range already reserved on this host; \ + pick a free start" + ), } - Ok(n) } - None if !cid_window_overlaps(DEFAULT_CID_START, occupied) => Ok(DEFAULT_CID_START), - None => { - let start = next_free_cid_block(occupied); - println!(" [ok] cid-start {start} (avoids CIDs already reserved by another VMM)"); + return Ok(start); + } + + // A recorded start already belongs to this install, so a re-run reuses it + // verbatim and never moves a live instance's pool out from under its running + // CVMs. It is deliberately not re-checked: `occupied` includes this install's + // own VMM, so the check could only ever fail against itself. + if let Some(start) = recorded { + return Ok(start); + } + + match first_free(candidates, occupied) { + Some(start) => { + println!(" [ok] cid-start {start} (avoids CIDs already reserved on this host)"); Ok(start) } + None => bail!( + "could not find a free CID window in {CID_PICK_ATTEMPTS} attempts; \ + pass --cid-start <start> explicitly" + ), } } @@ -59,26 +133,92 @@ mod tests { use super::*; #[test] - fn cid_default_when_range_free() { - assert_eq!(pick_cid_start(None, &[]).unwrap(), 1000); - // a pool entirely above the default window leaves it free. - assert_eq!(pick_cid_start(None, &[(2000, 3000)]).unwrap(), 1000); + fn takes_the_first_candidate_when_the_host_is_empty() { + let picked = pick_cid_start_from(None, None, &[], [40_000, 50_000].into_iter()).unwrap(); + assert_eq!(picked, 40_000); } #[test] - fn cid_auto_offsets_past_an_existing_vmm() { - // another VMM reserving [1000,2000) -> jump to 2000. - assert_eq!(pick_cid_start(None, &[(1000, 2000)]).unwrap(), 2000); - // its reserved pool plus a stray live CVM at 2500 -> jump past it. - assert_eq!( - pick_cid_start(None, &[(1000, 2000), (2500, 2501)]).unwrap(), - 3000 - ); + fn skips_candidates_whose_stride_is_taken() { + let occupied = [(40_000, 41_000), (50_000, 50_001)]; + let picked = + pick_cid_start_from(None, None, &occupied, [40_000, 50_000, 60_000].into_iter()) + .unwrap(); + assert_eq!(picked, 60_000); } #[test] - fn cid_explicit_honored_or_refused() { - assert_eq!(pick_cid_start(Some(2000), &[(1000, 2000)]).unwrap(), 2000); - assert!(pick_cid_start(Some(1000), &[(1000, 2000)]).is_err()); + fn a_candidate_needs_its_whole_stride_free_not_just_its_pool() { + // 45_000 is clear of [40_000, 41_000) as a 1000-wide pool, but it sits + // inside that candidate's stride, so the block is rejected. + let occupied = [(45_000, 45_001)]; + assert!(window_overlaps(40_000, CID_BLOCK_STRIDE, &occupied)); + assert!(!window_overlaps(40_000, CID_POOL_SIZE, &occupied)); + let picked = + pick_cid_start_from(None, None, &occupied, [40_000, 60_000].into_iter()).unwrap(); + assert_eq!(picked, 60_000); + } + + #[test] + fn gives_up_after_the_attempt_budget() { + let occupied = [(40_000, 50_000)]; + let err = pick_cid_start_from(None, None, &occupied, std::iter::repeat(40_000)) + .unwrap_err() + .to_string(); + assert!(err.contains("could not find a free CID window"), "{err}"); + } + + #[test] + fn explicit_is_honored_when_its_pool_is_free() { + // Overlaps the *stride* of an occupied block but not its pool, which an + // explicit choice is allowed to do. + let occupied = [(40_000, 41_000)]; + let picked = + pick_cid_start_from(Some(45_000), None, &occupied, [60_000].into_iter()).unwrap(); + assert_eq!(picked, 45_000); + } + + #[test] + fn explicit_is_refused_on_overlap_and_suggests_a_free_start() { + let occupied = [(40_000, 41_000)]; + let err = pick_cid_start_from(Some(40_500), None, &occupied, [60_000].into_iter()) + .unwrap_err() + .to_string(); + assert!(err.contains("--cid-start 40500 overlaps"), "{err}"); + assert!(err.contains("e.g. --cid-start 60000"), "{err}"); + } + + #[test] + fn explicit_is_refused_when_its_pool_runs_off_the_end_of_the_cid_space() { + let err = pick_cid_start_from(Some(u32::MAX - 10), None, &[], [60_000].into_iter()) + .unwrap_err() + .to_string(); + assert!(err.contains("leaves no room for a 1000-wide pool"), "{err}"); + } + + #[test] + fn explicit_wins_over_a_recorded_start() { + let picked = + pick_cid_start_from(Some(45_000), Some(70_000), &[], [60_000].into_iter()).unwrap(); + assert_eq!(picked, 45_000); + } + + #[test] + fn a_recorded_start_is_reused_verbatim() { + // The install's own VMM shows up in `occupied`; reusing must not treat + // that as a conflict, otherwise every re-run would move the pool. + let occupied = [(70_000, 71_000)]; + let picked = + pick_cid_start_from(None, Some(70_000), &occupied, [60_000].into_iter()).unwrap(); + assert_eq!(picked, 70_000); + } + + #[test] + fn random_starts_are_aligned_and_leave_room_for_a_stride() { + for start in random_starts().take(64) { + assert!((CID_MIN..=CID_MAX).contains(&start), "{start} out of range"); + assert_eq!(start % CID_BLOCK_STRIDE, 0, "{start} is not stride-aligned"); + assert!(start.checked_add(CID_BLOCK_STRIDE).is_some(), "{start}"); + } } } diff --git a/dstack/crates/dstackup/src/cli.rs b/dstack/crates/dstackup/src/cli.rs index 4d1b74689..cfcfbd927 100644 --- a/dstack/crates/dstackup/src/cli.rs +++ b/dstack/crates/dstackup/src/cli.rs @@ -11,7 +11,8 @@ pub(crate) const DEFAULT_VMM_BIN: &str = "dstack-vmm"; pub(crate) const DEFAULT_AUTH_BIN: &str = "dstack-auth"; pub(crate) const DEFAULT_SUPERVISOR_BIN: &str = "supervisor"; pub(crate) const DEFAULT_SOURCE_REPO: &str = "https://github.com/Dstack-TEE/dstack"; -pub(crate) const DEFAULT_SOURCE_REF: &str = "master"; +pub(crate) const DEFAULT_SOURCE_REF: &str = "next"; +pub(crate) const DEFAULT_RELEASE_API_BASE_URL: &str = "https://api.github.com/repos"; #[derive(Parser)] #[command(name = "dstackup", version, about = "set up and manage a dstack host")] @@ -21,6 +22,11 @@ pub(crate) struct Cli { #[arg(long, global = true)] pub(crate) host: Option<String>, + /// Base URL of the GitHub-compatible releases API, including its `/repos` + /// prefix. The owner/repository path is appended automatically. + #[arg(long, global = true, default_value = DEFAULT_RELEASE_API_BASE_URL)] + pub(crate) release_api_base_url: String, + #[command(subcommand)] pub(crate) command: Command, } diff --git a/dstack/crates/dstackup/src/destroy.rs b/dstack/crates/dstackup/src/destroy.rs index 2ff6ab4f3..bb6bcd18f 100644 --- a/dstack/crates/dstackup/src/destroy.rs +++ b/dstack/crates/dstackup/src/destroy.rs @@ -4,6 +4,7 @@ //! `dstackup destroy` — tear down what `install` started. +use crate::install::read_token_file; use crate::state::{read_state, state_path}; use crate::systemd::{remove_unit, systemctl, tool}; use anyhow::{Context, Result}; @@ -26,7 +27,8 @@ pub(crate) async fn cmd_destroy(prefix: Option<&str>, purge: bool) -> Result<()> // and the CVM qemu via the unit's cgroup. Look it up by recorded id // AND by name, so an install that died before persisting kms_vm_id // (or a torn state file) doesn't leave the CVM orphaned. - if let Ok(vmm) = Vmm::connect(&st.client_url) { + let token = read_token_file(Path::new(&st.client_token_path)); + if let Ok(vmm) = Vmm::connect_with_token(&st.client_url, token.as_deref()) { let mut target = st.kms_vm_id.clone(); if target.is_none() { if let Ok(s) = vmm.status().await { diff --git a/dstack/crates/dstackup/src/image.rs b/dstack/crates/dstackup/src/image.rs index 1874d31b9..91293b645 100644 --- a/dstack/crates/dstackup/src/image.rs +++ b/dstack/crates/dstackup/src/image.rs @@ -64,7 +64,7 @@ struct PullSpec { gpu: bool, } -pub(crate) async fn cmd_image(cmd: ImageCmd) -> Result<()> { +pub(crate) async fn cmd_image(cmd: ImageCmd, release_api_base_url: &str) -> Result<()> { match cmd { ImageCmd::Pull { version, @@ -75,7 +75,15 @@ pub(crate) async fn cmd_image(cmd: ImageCmd) -> Result<()> { } => { let image_dir = loc.dir(); validate_image_dir(&image_dir)?; - pull(version.as_deref(), gpu, &image_dir, force, insecure).await?; + pull( + version.as_deref(), + gpu, + &image_dir, + force, + insecure, + release_api_base_url, + ) + .await?; Ok(()) } ImageCmd::List { loc } => { @@ -98,6 +106,7 @@ pub(crate) async fn pull( image_dir: &str, force: bool, insecure: bool, + release_api_base_url: &str, ) -> Result<String> { println!( "dstackup image pull — {} image", @@ -107,7 +116,7 @@ pub(crate) async fn pull( "unified" } ); - let release = fetch_release(version).await?; + let release = fetch_release(version, release_api_base_url).await?; let asset = pick_asset(&release.assets, gpu).with_context(|| { format!( @@ -376,6 +385,7 @@ pub(crate) async fn resolve_or_pull_image( requested: Option<&str>, require: bool, required_files: &[&str], + release_api_base_url: &str, ) -> Result<Option<String>> { if let Some(name) = requested { if !valid_image_name(name) { @@ -391,7 +401,15 @@ pub(crate) async fn resolve_or_pull_image( } if let Some(spec) = pull_spec(name) { println!(" [..] image {name} not found locally; downloading it"); - let pulled = pull(Some(&spec.version), spec.gpu, image_dir, false, false).await?; + let pulled = pull( + Some(&spec.version), + spec.gpu, + image_dir, + false, + false, + release_api_base_url, + ) + .await?; ensure_image_has_required_files(image_dir, &pulled, required_files)?; return Ok(Some(pulled)); } @@ -459,7 +477,7 @@ pub(crate) async fn resolve_or_pull_image( } else { println!(" [..] no local guest image found; downloading the latest cpu image"); } - let pulled = pull(None, false, image_dir, false, false).await?; + let pulled = pull(None, false, image_dir, false, false, release_api_base_url).await?; if Path::new(image_dir) .join(&pulled) @@ -599,10 +617,10 @@ fn missing_named_image_message(image_dir: &str, name: &str) -> String { /// released from this monorepo. Do not probe the new repository first for old /// versions: the version boundary is authoritative and avoids redundant or /// misleading requests. -async fn fetch_release(version: Option<&str>) -> Result<Release> { +async fn fetch_release(version: Option<&str>, release_api_base_url: &str) -> Result<Release> { let client = reqwest::Client::new(); if let Some(version) = version { - let (version, url, releases_url) = tagged_release_location(version)?; + let (version, url, releases_url) = tagged_release_location(version, release_api_base_url)?; return fetch_tagged_release(&client, &url, releases_url) .await? .with_context(|| { @@ -610,7 +628,8 @@ async fn fetch_release(version: Option<&str>) -> Result<Release> { }); } - let list_url = format!("https://api.github.com/repos/{REPO}/releases?per_page=100"); + let api_base = release_api_base_url.trim().trim_end_matches('/'); + let list_url = format!("{api_base}/{REPO}/releases?per_page=100"); let releases: Vec<Release> = client .get(&list_url) .header("user-agent", "dstackup") @@ -630,13 +649,16 @@ async fn fetch_release(version: Option<&str>) -> Result<Release> { return Ok(release); } - let legacy_url = format!("https://api.github.com/repos/{LEGACY_REPO}/releases/latest"); + let legacy_url = format!("{api_base}/{LEGACY_REPO}/releases/latest"); fetch_tagged_release(&client, &legacy_url, LEGACY_RELEASES_URL) .await? .with_context(|| format!("no guest-OS release found; check {RELEASES_URL}")) } -fn tagged_release_location(version: &str) -> Result<(String, String, &'static str)> { +fn tagged_release_location( + version: &str, + release_api_base_url: &str, +) -> Result<(String, String, &'static str)> { let version = version .trim_start_matches(RELEASE_TAG_PREFIX) .trim_start_matches('v'); @@ -648,7 +670,10 @@ fn tagged_release_location(version: &str) -> Result<(String, String, &'static st }; Ok(( version.to_string(), - format!("https://api.github.com/repos/{repo}/releases/tags/{tag_prefix}{version}"), + format!( + "{}/{repo}/releases/tags/{tag_prefix}{version}", + release_api_base_url.trim().trim_end_matches('/') + ), releases_url, )) } @@ -701,9 +726,15 @@ async fn fetch_tagged_release( /// use the current unified image (which already contains conditional NVIDIA /// support). fn pick_asset(assets: &[Asset], gpu: bool) -> Option<&Asset> { + // `-dev` marks the dev *flavour*, which only ever appears as a name prefix + // (`dstack-dev-*`, `dstack-nvidia-dev-*`). Matching it as a bare substring + // would also reject release names whose version carries a `-dev` prerelease + // tag, e.g. `dstack-0.6.1-dev1.tar.gz`. + let is_dev_flavour = + |n: &str| n.starts_with("dstack-dev-") || n.starts_with("dstack-nvidia-dev-"); let matches = |a: &&Asset, want_legacy_gpu: bool| { let n = a.name.as_str(); - if !n.ends_with(".tar.gz") || n.ends_with("-uki.tar.gz") || n.contains("-dev") { + if !n.ends_with(".tar.gz") || n.ends_with("-uki.tar.gz") || is_dev_flavour(n) { return false; } let is_gpu = n.starts_with("dstack-nvidia-"); @@ -820,6 +851,41 @@ mod tests { ); } + #[test] + fn dev_flavour_is_matched_by_prefix_not_substring() { + // `-dev` as a *prerelease tag* is part of the version, not the flavour, + // so these releases must still be selectable. + let assets = vec![ + asset("dstack-dev-0.6.1-dev1.tar.gz"), + asset("dstack-0.6.1-dev1.tar.gz"), + asset("dstack-nvidia-dev-0.6.1-dev1.tar.gz"), + asset("dstack-nvidia-0.6.1-dev1.tar.gz"), + ]; + assert_eq!( + pick_asset(&assets, false).unwrap().name, + "dstack-0.6.1-dev1.tar.gz" + ); + assert_eq!( + pick_asset(&assets, true).unwrap().name, + "dstack-nvidia-0.6.1-dev1.tar.gz" + ); + } + + #[test] + fn prerelease_versions_are_selectable() { + for version in ["0.6.1-rc1", "0.6.1.rc1", "0.6.1-beta.2", "0.6.1.a1"] { + let assets = vec![ + asset(&format!("dstack-{version}-uki.tar.gz")), + asset(&format!("dstack-{version}.tar.gz")), + ]; + assert_eq!( + pick_asset(&assets, false).unwrap().name, + format!("dstack-{version}.tar.gz"), + "{version}" + ); + } + } + #[test] fn uki_archive_is_never_selected_as_a_host_image() { let assets = vec![asset("dstack-nvidia-0.6.0.a2-uki.tar.gz")]; @@ -830,7 +896,8 @@ mod tests { #[test] fn routes_pinned_releases_at_the_monorepo_boundary() { for version in ["0.5.11", "v0.5.11", "guest-os-v0.5.11"] { - let (normalized, url, releases_url) = tagged_release_location(version).unwrap(); + let (normalized, url, releases_url) = + tagged_release_location(version, crate::cli::DEFAULT_RELEASE_API_BASE_URL).unwrap(); assert_eq!(normalized, "0.5.11"); assert_eq!( url, @@ -840,7 +907,8 @@ mod tests { } for version in ["0.6.0", "0.6.0.a2", "1.0.0"] { - let (normalized, url, releases_url) = tagged_release_location(version).unwrap(); + let (normalized, url, releases_url) = + tagged_release_location(version, crate::cli::DEFAULT_RELEASE_API_BASE_URL).unwrap(); assert_eq!(normalized, version); assert_eq!( url, @@ -853,10 +921,23 @@ mod tests { #[test] fn rejects_versions_without_a_numeric_core() { for version in ["0.6", "latest", "0.x.0", "0.6.x"] { - assert!(tagged_release_location(version).is_err(), "{version}"); + assert!( + tagged_release_location(version, crate::cli::DEFAULT_RELEASE_API_BASE_URL).is_err(), + "{version}" + ); } } + #[test] + fn release_api_base_url_is_configurable_and_trailing_slash_safe() { + let (_, url, _) = + tagged_release_location("0.6.0", " http://127.0.0.1:1234/api/ ").unwrap(); + assert_eq!( + url, + "http://127.0.0.1:1234/api/Dstack-TEE/dstack/releases/tags/guest-os-v0.6.0" + ); + } + #[test] fn messages_mention_the_pull_command() { assert!(no_image_message("/d").contains("dstackup image pull")); diff --git a/dstack/crates/dstackup/src/install.rs b/dstack/crates/dstackup/src/install.rs index 469fc51ae..b32f9fea0 100644 --- a/dstack/crates/dstackup/src/install.rs +++ b/dstack/crates/dstackup/src/install.rs @@ -28,17 +28,17 @@ const DAEMON_BINARIES: &[(&str, &str)] = &[ ("supervisor", "supervisor"), ]; -pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { - // --expose is not safe yet: the rendered vmm.toml binds the VM-control - // plane with neither TLS nor an auth token (the management RPCs are not - // behind an auth guard), so exposing it would hand deploy/destroy to anyone - // who can reach the IP. Refuse until the TLS+token transport lands; the - // supported path is localhost + an SSH tunnel. +pub(crate) async fn cmd_install(mut o: InstallOpts, release_api_base_url: &str) -> Result<()> { + // --expose is not safe yet: the rendered vmm.toml now gates the management + // API behind a generated token, but the transport is still plain HTTP, so + // exposing it would send that bearer token in cleartext to anyone on-path. + // Refuse until the TLS transport lands; the supported path is localhost + an + // SSH tunnel. if let Some(ip) = &o.expose { bail!( "--expose {ip} is not yet safe: it would bind the VM-control plane on \ - {ip}:{port} with no TLS and no auth. reach the dashboard over an SSH \ - tunnel instead: ssh -L {port}:127.0.0.1:{port} <host>", + {ip}:{port} over plain HTTP, leaking the API token in cleartext. reach \ + the dashboard over an SSH tunnel instead: ssh -L {port}:127.0.0.1:{port} <host>", port = o.dashboard_port ); } @@ -81,10 +81,24 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { let client_url = format!("http://{bind}:{}", o.dashboard_port); let kms_port = resolve_kms_port(&o, &st)?; + // management-API token: reuse the one a prior install wrote (so re-runs and + // an already-running VMM keep matching credentials), else mint a fresh one + // below once the config dir exists. The existing token authenticates the + // preflight probe against an already-running, auth-enabled VMM. + let token_path = layout.config_dir.join("vmm-auth-token"); + let existing_token = read_token_file(&token_path); + // 4. preflight - fail BEFORE any side effect (image download, key provider, // dirs, units), so a CID/port clash can't half-install the host. - let cid_start = pick_cid_start(o.cid_start, &host::occupied_cid_ranges())?; - let kms_owned = kms_port_owned(&st, &client_url, kms_port, o.no_kms).await; + let cid_start = pick_cid_start(o.cid_start, st.cid_start, &host::occupied_cid_ranges())?; + let kms_owned = kms_port_owned( + &st, + &client_url, + existing_token.as_deref(), + kms_port, + o.no_kms, + ) + .await; let port_plan = tcp_port_plan(&o, &st, platform, &bind, &client_url, kms_port, kms_owned); preflight_ports(&port_plan)?; @@ -99,6 +113,7 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { o.image.as_deref(), !o.no_kms, required_image_files, + release_api_base_url, ) .await?; let os_image_hash = resolve_image_pin(&o, &images, platform)?; @@ -117,6 +132,15 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; } + // ensure a management-API token exists on disk (0600) before rendering the + // config that references it. The local `dstack` CLI reads this path from the + // install state, so it authenticates automatically. + let vmm_token = match existing_token { + Some(t) => t, + None => generate_vmm_token()?, + }; + write_token_file(&token_path, &vmm_token)?; + // 8. resolve the key provider - run our own unless told to use an existing // one (TDX only; SNP has no SGX local provider). let (kp_addr, kp_port, kp_own_project) = @@ -144,6 +168,8 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { key_provider_port: kp_port as u32, kms_urls: kms_urls.clone(), platform, + auth_enabled: true, + auth_token: vmm_token.clone(), ..Default::default() }); // the KMS-in-CVM reaches the host auth webhook at 10.0.2.2:<auth_port>. @@ -191,7 +217,9 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { st.run_dir = layout.run_dir.display().to_string(); st.allowlist_path = allow_path.display().to_string(); st.client_url = client_url.clone(); + st.client_token_path = token_path.display().to_string(); st.auth_port = o.auth_port; + st.cid_start = Some(cid_start); st.platform = platform.vmm_str().to_string(); st.image = o.image.clone(); let instance = effective_instance(&o, &layout, explicit_prefix); @@ -215,7 +243,7 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { st.auth_unit = auth_unit.clone(); // 11. VMM systemd unit (idempotent). - if vmm_reachable(&client_url).await { + if vmm_reachable(&client_url, Some(&vmm_token)).await { println!(" [ok] VMM already serving at {client_url}"); } else { install_unit( @@ -225,7 +253,7 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { .context("installing the VMM unit")?; println!(" [ok] started {vmm_unit}.service"); print!(" [..] waiting for VMM at {client_url} "); - if wait_ready(&client_url, Duration::from_secs(25)).await { + if wait_ready(&client_url, Some(&vmm_token), Duration::from_secs(25)).await { println!("=> ready"); } else { println!("=> not ready within timeout (journalctl -u {vmm_unit})"); @@ -241,7 +269,7 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { if o.no_kms { println!(" (--no-kms: skipping KMS deploy)"); } else { - let vmm = Vmm::connect(&client_url)?; + let vmm = Vmm::connect_with_token(&client_url, Some(&vmm_token))?; let existing = match &st.kms_vm_id { Some(id) if vmm.has_vm(id).await => Some(id.clone()), _ => None, @@ -266,6 +294,9 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { host_address: "127.0.0.1".into(), host_port: kms_port as u32, vm_port: 8000, + // Unpinned: this deploys the node default topology, which + // is one NIC, and the VMM resolves that itself. + nic_index: None, }], ..Default::default() }; @@ -1007,7 +1038,13 @@ fn tcp_port_plan( ports } -async fn kms_port_owned(st: &State, client_url: &str, kms_port: u16, no_kms: bool) -> bool { +async fn kms_port_owned( + st: &State, + client_url: &str, + token: Option<&str>, + kms_port: u16, + no_kms: bool, +) -> bool { if no_kms || st.client_url != client_url || state_kms_port(st) != Some(kms_port) @@ -1019,7 +1056,7 @@ async fn kms_port_owned(st: &State, client_url: &str, kms_port: u16, no_kms: boo let Some(kms_vm_id) = &st.kms_vm_id else { return false; }; - match Vmm::connect(client_url) { + match Vmm::connect_with_token(client_url, token) { Ok(vmm) => vmm.has_vm(kms_vm_id).await, Err(_) => false, } @@ -1158,18 +1195,18 @@ fn kms_vm_boot_state( } /// one-shot liveness probe of the VMM. -async fn vmm_reachable(client_url: &str) -> bool { - match Vmm::connect(client_url) { +async fn vmm_reachable(client_url: &str, token: Option<&str>) -> bool { + match Vmm::connect_with_token(client_url, token) { Ok(vmm) => vmm.status().await.is_ok(), Err(_) => false, } } /// poll the VMM `Status` RPC until it succeeds or the deadline passes. -async fn wait_ready(client_url: &str, timeout: Duration) -> bool { +async fn wait_ready(client_url: &str, token: Option<&str>, timeout: Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; loop { - if let Ok(vmm) = Vmm::connect(client_url) { + if let Ok(vmm) = Vmm::connect_with_token(client_url, token) { if vmm.status().await.is_ok() { return true; } @@ -1181,6 +1218,30 @@ async fn wait_ready(client_url: &str, timeout: Duration) -> bool { } } +/// mint a 256-bit management-API token, hex-encoded, from the OS RNG. +fn generate_vmm_token() -> Result<String> { + let mut buf = [0u8; 32]; + let mut f = fs::File::open("/dev/urandom").context("opening /dev/urandom")?; + std::io::Read::read_exact(&mut f, &mut buf).context("reading /dev/urandom")?; + Ok(hex::encode(buf)) +} + +/// read a previously written management-API token, if the file exists and is +/// non-empty. +pub(crate) fn read_token_file(path: &Path) -> Option<String> { + let token = fs::read_to_string(path).ok()?; + let token = token.trim(); + (!token.is_empty()).then(|| token.to_string()) +} + +/// write the management-API token atomically with owner-only (0600) +/// permissions — it is a bearer credential, so it is created 0600 up front +/// (never exposed with wider bits, even transiently). +fn write_token_file(path: &Path, token: &str) -> Result<()> { + dstack_cli_core::fsutil::write_atomic_mode(path, token, 0o600) + .with_context(|| format!("writing {}", path.display())) +} + #[cfg(test)] mod tests { use super::*; diff --git a/dstack/crates/dstackup/src/main.rs b/dstack/crates/dstackup/src/main.rs index 42082685c..9529231e5 100644 --- a/dstack/crates/dstackup/src/main.rs +++ b/dstack/crates/dstackup/src/main.rs @@ -38,8 +38,8 @@ async fn main() -> Result<()> { let platform = default_platform(prefix.as_deref()).or_else(host::Platform::detect); cmd_status(&host, platform).await } - Command::Install(opts) => install::cmd_install(opts).await, - Command::Image(cmd) => image::cmd_image(cmd).await, + Command::Install(opts) => install::cmd_install(opts, &cli.release_api_base_url).await, + Command::Image(cmd) => image::cmd_image(cmd, &cli.release_api_base_url).await, Command::Destroy { prefix, purge } => destroy::cmd_destroy(prefix.as_deref(), purge).await, } } diff --git a/dstack/crates/dstackup/src/state.rs b/dstack/crates/dstackup/src/state.rs index 3414ac818..aa075b985 100644 --- a/dstack/crates/dstackup/src/state.rs +++ b/dstack/crates/dstackup/src/state.rs @@ -30,7 +30,16 @@ pub(crate) struct State { #[serde(default)] pub(crate) platform: String, pub(crate) client_url: String, + /// path to the management-API bearer token file the local `dstack` CLI + /// reads to authenticate against the VMM. + #[serde(default)] + pub(crate) client_token_path: String, pub(crate) auth_port: u16, + /// vsock CID pool start this install chose, so a re-run reuses it instead of + /// picking a fresh window and moving a live instance's pool. Absent in state + /// files written before this was recorded. + #[serde(default)] + pub(crate) cid_start: Option<u32>, /// systemd unit names (without the `.service` suffix). #[serde(default)] pub(crate) vmm_unit: String, @@ -67,3 +76,26 @@ pub(crate) fn write(path: &Path, body: &str) -> Result<()> { dstack_cli_core::fsutil::write_atomic(path, body) .with_context(|| format!("writing {}", path.display())) } + +#[cfg(test)] +mod tests { + use super::*; + + /// `read_state` swallows a parse error into `None`, which an install reads as + /// "nothing here" — it would then re-pick the CID window and ports of a live + /// install. So a state file written before a field existed must still load. + #[test] + fn a_state_file_without_cid_start_still_loads() { + let dir = tempfile::tempdir().unwrap(); + let body = r#"{ + "prefix": "/var/lib/dstack", + "client_url": "http://127.0.0.1:9080", + "auth_port": 8090 + }"#; + fs::write(state_path(dir.path()), body).unwrap(); + + let st = read_state(dir.path()).expect("legacy state file must still parse"); + assert_eq!(st.cid_start, None); + assert_eq!(st.auth_port, 8090); + } +} diff --git a/dstack/crates/mock-attestation/Cargo.toml b/dstack/crates/mock-attestation/Cargo.toml new file mode 100644 index 000000000..65091e6ae --- /dev/null +++ b/dstack/crates/mock-attestation/Cargo.toml @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "mock-attestation" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +description = "Development-only cryptographically valid attestation evidence generator" + +[[bin]] +name = "dstack-mock-attestation" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +fs-err.workspace = true +rcgen.workspace = true +ciborium.workspace = true +p384 = { workspace = true, features = ["ecdsa", "pkcs8"] } +p256 = { workspace = true, features = ["ecdsa", "pkcs8"] } +sha2.workspace = true +sev.workspace = true +sev-snp-qvl.workspace = true +tpm-types.workspace = true +tpm-qvl.workspace = true +dstack-types.workspace = true +dcap-qvl.workspace = true +scale.workspace = true +yasna.workspace = true +chrono.workspace = true +rand.workspace = true +tokio = { workspace = true, features = ["full"] } +axum = "0.8" +hex.workspace = true +urlencoding = "2" +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +time.workspace = true + +[dev-dependencies] +tempfile.workspace = true +nsm-qvl.workspace = true diff --git a/dstack/crates/mock-attestation/README.md b/dstack/crates/mock-attestation/README.md new file mode 100644 index 000000000..4ac04abf8 --- /dev/null +++ b/dstack/crates/mock-attestation/README.md @@ -0,0 +1,90 @@ +# mock-attestation + +Development-only, cryptographically valid attestation evidence for CI and the +`dstack-tee-simulator` dev image. + +The crate derives a development PKI from a 32-byte seed and dynamically signs evidence for: + +- Intel TDX/DCAP, including TCB Info, QE Identity, PCK certificates and CRLs; +- AMD SEV-SNP, including ARK, ASK, VCEK and reports; +- TPM, including AK certificates, quotes, AIA and CRLs; +- AWS NSM, including the certificate bundle and COSE Sign1 document. + +All evidence is verified in the test suite by the production QVLs. These keys +are test credentials and must never be copied into a production image. + +## CLI + +Generate matching public roots and `tee-simulator.json`: + +```console +dstack-mock-attestation generate \ + --output ./mock-roots \ + --collateral-base-url http://HOST_REACHABLE_FROM_VERIFIER:8088 +``` + +Use the generated simulator config for both the guest-side simulator and the +host-side Mock PCCS/KDS/AIA service: + +```console +dstack-mock-attestation serve \ + --listen 127.0.0.1:8088 \ + --config ./mock-roots/tee-simulator.json \ + --output ./active-mock-roots +``` + +The server exposes only production-shaped public collateral endpoints. Evidence +is generated inside the development guest by `dstack-tee-simulator`; the HTTP +service deliberately provides no unauthenticated signing endpoint. + +## Dev image + +Select the platform in the development-only `.tee-simulator.json`; omission +defaults to TDX: + +```json +{ + "platform": "dstack-amd-sev-snp", + "mock_attestation_seed": "<64 hex characters>", + "collateral_base_url": "http://HOST_REACHABLE_FROM_VERIFIER:8088" +} +``` + +Valid values mirror the supported attestation modes: `dstack-tdx`, +`dstack-gcp-tdx`, `dstack-amd-sev-snp`, `dstack-nitro-enclave`, and +`dstack-aws-nitro-tpm`. The simulator exposes +the production guest ABI for the selected platform (TSM configfs, vTPM, or an +NSM CUSE character device); attester libraries contain no mock HTTP or +environment-variable path. The guest only reads the +seed from `.tee-simulator.json`; it never writes credentials or roots back into +`/dstack/.host-shared`. CI retains the generated public roots and mounts them +into verifier/KMS/gateway. The independently running host collateral service +reconstructs the same hierarchy from the seed. Configure it under +`[attestation.urls]`: TDX uses `pccs`, and SEV-SNP uses `amd_kds`. + +The guest needs those roots too, to verify the KMS and the gateway it talks to. +They do not travel from the host: `dstack-tee-simulator` derives them from the +same seed and writes them to `/run/dstack/attestation`, guest tmpfs the host +cannot reach, before `dstack-prepare` starts. `dstack-util` reads that one +directory and nothing else, so a host can never nominate the trust anchor that +authenticates its guest's key provider. Only the development image ships the +simulator, and image contents are measured, so on a production image the +directory never exists and vendor production roots are the only outcome. + +Every service configured with a mock root through its own TOML — KMS, gateway, +`dstack-verifier` — must also explicitly set +`attestation.insecure_allow_external_trust_anchors = true`. Merely mounting and +configuring a mock root is rejected at startup while this flag remains false. +The flag exists to make an operator acknowledge a hand-written non-production +root, so it has no counterpart in the guest handoff above, where one program +writes the roots and the next reads them out of a directory it authenticates. + +The seed adds only 64 hex bytes (the simulator config is well below 1 KiB). + +## Required negative tests + +The crate tests ensure every platform rejects a different root and modified +signed bytes. SEV-SNP additionally performs expected `report_data` comparison +inside its QVL API. TDX/TPM/NSM expose their authenticated binding field for +the caller; the production `dstack-attest` layer performs the final equality +check. diff --git a/dstack/crates/mock-attestation/src/lib.rs b/dstack/crates/mock-attestation/src/lib.rs new file mode 100644 index 000000000..0c84d0eb5 --- /dev/null +++ b/dstack/crates/mock-attestation/src/lib.rs @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Development-only attestation PKI and evidence generation. +//! +//! Private keys produced by this crate are test material and must never be +//! installed in a production image. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use p256::pkcs8::EncodePrivateKey; +use rcgen::KeyPair; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256, Sha384}; + +pub mod nsm; +pub mod server; + +/// Fail closed when authenticated evidence is not bound to the caller's +/// challenge/report data. +pub fn ensure_report_data(actual: &[u8], expected: &[u8]) -> Result<()> { + anyhow::ensure!(actual == expected, "mock attestation report_data mismatch"); + Ok(()) +} +pub mod sev_snp; +pub mod tdx; +pub mod tpm; + +pub const MOCK_SEED_LEN: usize = 32; + +/// Returns whether the simulated platform provides its own TPM device. +pub fn platform_provides_tpm(platform: dstack_types::TeeVariant) -> bool { + matches!( + platform, + dstack_types::TeeVariant::DstackGcpTdx | dstack_types::TeeVariant::DstackAwsNitroTpm + ) +} + +pub fn parse_seed(value: &str) -> Result<[u8; MOCK_SEED_LEN]> { + let bytes = hex::decode(value).context("mock attestation seed must be hex")?; + bytes + .try_into() + .map_err(|_| anyhow::anyhow!("mock attestation seed must be 32 bytes")) +} + +pub fn random_seed() -> [u8; MOCK_SEED_LEN] { + rand::random() +} + +pub(crate) fn p256_key(seed: &[u8; 32], label: &str) -> Result<KeyPair> { + for counter in 0u32.. { + let bytes = Sha256::new() + .chain_update(b"dstack-mock-p256-v1") + .chain_update(seed) + .chain_update(label) + .chain_update(counter.to_be_bytes()) + .finalize(); + if let Ok(key) = p256::ecdsa::SigningKey::from_slice(&bytes) { + return Ok(KeyPair::from_pem(&key.to_pkcs8_pem(Default::default())?)?); + } + } + unreachable!() +} + +pub(crate) fn p384_key(seed: &[u8; 32], label: &str) -> Result<KeyPair> { + for counter in 0u32.. { + let bytes = Sha384::new() + .chain_update(b"dstack-mock-p384-v1") + .chain_update(seed) + .chain_update(label) + .chain_update(counter.to_be_bytes()) + .finalize(); + if let Ok(key) = p384::ecdsa::SigningKey::from_slice(&bytes) { + return Ok(KeyPair::from_pem(&key.to_pkcs8_pem(Default::default())?)?); + } + } + unreachable!() +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AssetManifest { + pub version: u32, + pub tdx_root_ca: PathBuf, + pub tpm_root_ca: PathBuf, + pub nsm_root_ca: PathBuf, + pub sev_snp_milan_root_ca: PathBuf, + pub sev_snp_genoa_root_ca: PathBuf, + pub sev_snp_turin_root_ca: PathBuf, +} + +impl AssetManifest { + pub fn write_to(&self, output: &Path) -> Result<()> { + fs_err::write( + output.join("manifest.json"), + serde_json::to_vec_pretty(self).context("failed to serialize asset manifest")?, + )?; + Ok(()) + } +} + +pub fn generate_assets(output: &Path) -> Result<AssetManifest> { + generate_assets_from_seed(output, random_seed(), "http://127.0.0.1:8088") +} + +pub fn generate_assets_from_seed( + output: &Path, + seed: [u8; 32], + base_url: &str, +) -> Result<AssetManifest> { + fs_err::create_dir_all(output)?; + let state = server::MockCollateralState::from_seed(seed, base_url)?; + state.write_roots(output)?; + let tdx = PathBuf::from("tdx-root-ca.pem"); + let tpm = PathBuf::from("tpm-root-ca.pem"); + let nsm = PathBuf::from("nsm-root-ca.pem"); + let milan = PathBuf::from("sev-snp-root-ca.pem"); + let genoa = milan.clone(); + let turin = milan.clone(); + let manifest = AssetManifest { + version: 1, + tdx_root_ca: tdx, + tpm_root_ca: tpm, + nsm_root_ca: nsm, + sev_snp_milan_root_ca: milan, + sev_snp_genoa_root_ca: genoa, + sev_snp_turin_root_ca: turin, + }; + manifest.write_to(output)?; + fs_err::write( + output.join("tee-simulator.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "platform": "dstack-tdx", "mock_attestation_seed": hex::encode(seed), "collateral_base_url": base_url + }))?, + )?; + Ok(manifest) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generates_all_platform_roots() { + let dir = tempfile::tempdir().unwrap(); + let manifest = generate_assets(dir.path()).unwrap(); + for path in [ + manifest.tdx_root_ca, + manifest.tpm_root_ca, + manifest.nsm_root_ca, + manifest.sev_snp_milan_root_ca, + manifest.sev_snp_genoa_root_ca, + manifest.sev_snp_turin_root_ca, + ] { + assert!(dir.path().join(path).exists()); + } + } + + #[test] + fn identifies_platforms_that_provide_a_tpm() { + assert!(platform_provides_tpm( + dstack_types::TeeVariant::DstackGcpTdx + )); + assert!(platform_provides_tpm( + dstack_types::TeeVariant::DstackAwsNitroTpm + )); + assert!(!platform_provides_tpm(dstack_types::TeeVariant::DstackTdx)); + } +} diff --git a/dstack/crates/mock-attestation/src/main.rs b/dstack/crates/mock-attestation/src/main.rs new file mode 100644 index 000000000..03c8b10d6 --- /dev/null +++ b/dstack/crates/mock-attestation/src/main.rs @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +use std::path::PathBuf; + +use anyhow::Result; +use clap::{Parser, Subcommand}; + +#[derive(Parser)] +#[command(about = "Generate development-only mock attestation assets")] +struct Args { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Generate PKI roots and a manifest for every supported platform. + Generate { + #[arg(long)] + output: PathBuf, + #[arg(long, default_value = "http://127.0.0.1:8088")] + collateral_base_url: String, + }, + /// Serve mock PCCS, AMD KDS and certificate collateral endpoints. + Serve { + #[arg(long, default_value = "127.0.0.1:8088")] + listen: std::net::SocketAddr, + /// Write the roots matching this server instance into this directory. + #[arg(long)] + output: Option<PathBuf>, + /// Read the seed and collateral URL from a simulator config JSON file. + #[arg(long)] + config: Option<PathBuf>, + }, +} + +#[tokio::main] +async fn main() -> Result<()> { + match Args::parse().command { + Command::Generate { + output, + collateral_base_url, + } => { + mock_attestation::generate_assets_from_seed( + &output, + mock_attestation::random_seed(), + &collateral_base_url, + )?; + } + Command::Serve { + listen, + output, + config, + } => { + let state = if let Some(path) = config { + let config: dstack_types::TeeSimulatorConfig = + serde_json::from_slice(&fs_err::read(path)?)?; + let seed = mock_attestation::parse_seed( + config + .mock_attestation_seed + .as_deref() + .ok_or_else(|| anyhow::anyhow!("mock_attestation_seed missing"))?, + )?; + let url = config + .collateral_base_url + .as_deref() + .unwrap_or("http://127.0.0.1:8088"); + std::sync::Arc::new(mock_attestation::server::MockCollateralState::from_seed( + seed, url, + )?) + } else { + std::sync::Arc::new(mock_attestation::server::MockCollateralState::new()?) + }; + if let Some(output) = output { + state.write_roots(&output)?; + } + mock_attestation::server::serve(listen, state).await?; + } + } + Ok(()) +} diff --git a/dstack/crates/mock-attestation/src/nsm.rs b/dstack/crates/mock-attestation/src/nsm.rs new file mode 100644 index 000000000..14831c7c9 --- /dev/null +++ b/dstack/crates/mock-attestation/src/nsm.rs @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeMap; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use p384::ecdsa::{signature::hazmat::PrehashSigner, Signature, SigningKey}; +use p384::pkcs8::DecodePrivateKey; +use rcgen::{ + BasicConstraints, Certificate, CertificateParams, CertifiedKey, DnType, IsCa, KeyPair, + KeyUsagePurpose, +}; +use serde::Serialize; +use sha2::{Digest, Sha384}; +use time::{Duration, OffsetDateTime}; + +/// In-memory NSM test hierarchy. A fresh leaf is issued so validity follows CI +/// wall-clock time rather than a checked-in fixture. +pub struct NsmGenerator { + root: Certificate, + root_key: KeyPair, + leaf: Certificate, + leaf_key: KeyPair, +} + +#[derive(Serialize)] +struct Document { + module_id: String, + digest: String, + timestamp: u64, + pcrs: BTreeMap<u16, Vec<u8>>, + certificate: Vec<u8>, + cabundle: Vec<Vec<u8>>, + public_key: Option<Vec<u8>>, + user_data: Option<Vec<u8>>, + nonce: Option<Vec<u8>>, +} + +impl NsmGenerator { + pub fn new() -> Result<Self> { + Self::from_seed(rand::random()) + } + + pub fn from_seed(seed: [u8; 32]) -> Result<Self> { + let CertifiedKey { + cert: root, + key_pair: root_key, + } = make_root(&seed)?; + let leaf_key = crate::p384_key(&seed, "nsm-leaf")?; + let mut params = CertificateParams::new(Vec::<String>::new())?; + params + .distinguished_name + .push(DnType::CommonName, "mock.nsm.dstack"); + params.key_usages.push(KeyUsagePurpose::DigitalSignature); + let (not_before, not_after) = validity(); + params.not_before = not_before; + params.not_after = not_after; + let leaf = params.signed_by(&leaf_key, &root, &root_key)?; + Ok(Self { + root, + root_key, + leaf, + leaf_key, + }) + } + + pub fn root_ca_pem(&self) -> String { + self.root.pem() + } + + pub fn root_key_pem(&self) -> String { + self.root_key.serialize_pem() + } + + pub fn attest(&self, report_data: &[u8]) -> Result<Vec<u8>> { + let pcrs = (0..=2) + .map(|index| (index, vec![index as u8; 48])) + .collect(); + self.attest_with_pcrs(report_data, pcrs) + } + + pub fn attest_with_pcrs( + &self, + report_data: &[u8], + pcrs: BTreeMap<u16, Vec<u8>>, + ) -> Result<Vec<u8>> { + self.attest_with_claims(Some(report_data), None, None, pcrs) + } + + pub fn attest_with_claims( + &self, + user_data: Option<&[u8]>, + nonce: Option<&[u8]>, + public_key: Option<&[u8]>, + pcrs: BTreeMap<u16, Vec<u8>>, + ) -> Result<Vec<u8>> { + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system clock before UNIX epoch")? + .as_millis() as u64; + let document = Document { + module_id: "mock-nsm".into(), + digest: "SHA384".into(), + timestamp, + pcrs, + certificate: self.leaf.der().to_vec(), + cabundle: vec![self.root.der().to_vec()], + public_key: public_key.map(ToOwned::to_owned), + user_data: user_data.map(ToOwned::to_owned), + nonce: nonce.map(ToOwned::to_owned), + }; + let mut payload = Vec::new(); + ciborium::into_writer(&document, &mut payload)?; + + let mut protected = Vec::new(); + let protected_map = BTreeMap::from([(1i64, ciborium::Value::Integer((-35).into()))]); + ciborium::into_writer(&protected_map, &mut protected)?; + let sig_structure = ciborium::Value::Array(vec![ + ciborium::Value::Text("Signature1".into()), + ciborium::Value::Bytes(protected.clone()), + ciborium::Value::Bytes(Vec::new()), + ciborium::Value::Bytes(payload.clone()), + ]); + let mut to_sign = Vec::new(); + ciborium::into_writer(&sig_structure, &mut to_sign)?; + let digest = Sha384::digest(to_sign); + let signing_key = SigningKey::from_pkcs8_pem(&self.leaf_key.serialize_pem())?; + let signature: Signature = signing_key.sign_prehash(&digest)?; + + let cose = ciborium::Value::Array(vec![ + ciborium::Value::Bytes(protected), + ciborium::Value::Map(Vec::new()), + ciborium::Value::Bytes(payload), + ciborium::Value::Bytes(signature.to_bytes().to_vec()), + ]); + let mut encoded = Vec::new(); + ciborium::into_writer(&cose, &mut encoded)?; + Ok(encoded) + } +} + +fn make_root(seed: &[u8; 32]) -> Result<CertifiedKey> { + let key_pair = crate::p384_key(seed, "nsm-root")?; + let mut params = CertificateParams::new(Vec::<String>::new())?; + params + .distinguished_name + .push(DnType::CommonName, "Mock AWS Nitro Enclaves Root CA"); + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + params.key_usages.extend([ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + ]); + let (not_before, not_after) = validity(); + params.not_before = not_before; + params.not_after = not_after; + let cert = params.self_signed(&key_pair)?; + Ok(CertifiedKey { cert, key_pair }) +} + +fn validity() -> (OffsetDateTime, OffsetDateTime) { + let now = OffsetDateTime::now_utc(); + (now - Duration::days(1), now + Duration::days(30)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generated_document_passes_real_qvl_and_negative_cases_fail() { + let generator = NsmGenerator::new().unwrap(); + let report_data = [0x42; 64]; + let evidence = generator.attest(&report_data).unwrap(); + let verifier = nsm_qvl::QuoteVerifier::new(generator.root_ca_pem()); + let verified = verifier.verify(&evidence, None, None).unwrap(); + assert_eq!(verified.user_data.as_deref(), Some(report_data.as_slice())); + + let wrong = NsmGenerator::new().unwrap(); + assert!(nsm_qvl::QuoteVerifier::new(wrong.root_ca_pem()) + .verify(&evidence, None, None) + .is_err()); + + let mut tampered = evidence; + *tampered.last_mut().unwrap() ^= 1; + assert!(verifier.verify(&tampered, None, None).is_err()); + assert!( + crate::ensure_report_data(verified.user_data.as_deref().unwrap(), &[0x24; 64]).is_err() + ); + } +} diff --git a/dstack/crates/mock-attestation/src/server.rs b/dstack/crates/mock-attestation/src/server.rs new file mode 100644 index 000000000..35d0216a0 --- /dev/null +++ b/dstack/crates/mock-attestation/src/server.rs @@ -0,0 +1,389 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +use std::net::SocketAddr; +use std::sync::Arc; + +use anyhow::Result; +use axum::{ + body::Body, + extract::State, + http::{header::CONTENT_TYPE, HeaderValue, Response, StatusCode}, + response::IntoResponse, + routing::get, + Router, +}; +use serde_json::json; + +use crate::{nsm::NsmGenerator, sev_snp::SevSnpGenerator, tdx::TdxGenerator, tpm::TpmGenerator}; + +pub struct MockCollateralState { + pub tdx: Arc<TdxGenerator>, + pub sev_snp: Arc<SevSnpGenerator>, + pub tpm: Arc<TpmGenerator>, + pub nsm: Arc<NsmGenerator>, +} + +impl MockCollateralState { + pub fn new() -> Result<Self> { + Self::with_base_url("http://127.0.0.1:8088") + } + + pub fn with_base_url(base_url: &str) -> Result<Self> { + Self::from_seed(rand::random(), base_url) + } + + pub fn from_seed(seed: [u8; 32], base_url: &str) -> Result<Self> { + Ok(Self { + tdx: Arc::new(TdxGenerator::from_seed(seed)?), + sev_snp: Arc::new(SevSnpGenerator::from_seed(seed)?), + tpm: Arc::new(TpmGenerator::from_seed(seed, base_url)?), + nsm: Arc::new(NsmGenerator::from_seed(seed)?), + }) + } + + pub fn write_roots(&self, output: &std::path::Path) -> Result<()> { + fs_err::create_dir_all(output)?; + fs_err::write(output.join("tdx-root-ca.pem"), self.tdx.root_ca_pem())?; + fs_err::write( + output.join("sev-snp-root-ca.pem"), + self.sev_snp.root_ca_pem(), + )?; + fs_err::write(output.join("tpm-root-ca.pem"), self.tpm.root_ca_pem())?; + fs_err::write(output.join("nsm-root-ca.pem"), self.nsm.root_ca_pem())?; + Ok(()) + } +} + +pub fn router(state: Arc<MockCollateralState>) -> Router { + Router::new() + .route("/sgx/certification/v4/pckcrl", get(pck_crl)) + .route("/sgx/certification/v4/rootcacrl", get(pccs_root_crl)) + // Older guest images use the SGX PCS prefix for shared DCAP collateral. + // Serve both spellings so simulated TDX evidence remains compatible. + .route("/sgx/certification/v4/tcb", get(tcb_info)) + .route("/sgx/certification/v4/qe/identity", get(qe_identity)) + .route("/tdx/certification/v4/tcb", get(tcb_info)) + .route("/tdx/certification/v4/qe/identity", get(qe_identity)) + .route("/vcek/v1/Milan/cert_chain", get(sev_ca_chain)) + .route("/vcek/v1/Milan/{chip_id}", get(sev_vcek)) + .route("/tpm/aia/root.pem", get(tpm_root)) + .route("/tpm/aia/intermediate.der", get(tpm_intermediate)) + .route("/tpm/crl/root.crl", get(tpm_root_crl)) + .route("/tpm/crl/intermediate.crl", get(tpm_intermediate_crl)) + .with_state(state) +} + +pub async fn serve(addr: SocketAddr, state: Arc<MockCollateralState>) -> Result<()> { + let listener = tokio::net::TcpListener::bind(addr).await?; + serve_listener(listener, state).await +} + +pub async fn serve_listener( + listener: tokio::net::TcpListener, + state: Arc<MockCollateralState>, +) -> Result<()> { + axum::serve(listener, router(state)).await?; + Ok(()) +} + +async fn pccs_root_crl(State(state): State<Arc<MockCollateralState>>) -> impl IntoResponse { + binary(hex::encode(state.tdx.root_crl_der()).into_bytes(), None) +} + +async fn pck_crl(State(state): State<Arc<MockCollateralState>>) -> impl IntoResponse { + let Ok(collateral) = state.tdx.sample_collateral() else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + binary( + state.tdx.pck_crl_der(), + Some(("SGX-PCK-CRL-Issuer-Chain", collateral.pck_crl_issuer_chain)), + ) +} + +async fn tcb_info(State(state): State<Arc<MockCollateralState>>) -> impl IntoResponse { + let Ok(collateral) = state.tdx.sample_collateral() else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let Ok(tcb_info) = serde_json::from_str::<serde_json::Value>(&collateral.tcb_info) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + json_response( + json!({"tcbInfo": tcb_info, "signature": hex::encode(collateral.tcb_info_signature)}), + Some(( + "SGX-TCB-Info-Issuer-Chain", + collateral.tcb_info_issuer_chain, + )), + ) +} + +async fn qe_identity(State(state): State<Arc<MockCollateralState>>) -> impl IntoResponse { + let Ok(collateral) = state.tdx.sample_collateral() else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let Ok(qe_identity) = serde_json::from_str::<serde_json::Value>(&collateral.qe_identity) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + json_response( + json!({"enclaveIdentity": qe_identity, "signature": hex::encode(collateral.qe_identity_signature)}), + Some(( + "SGX-Enclave-Identity-Issuer-Chain", + collateral.qe_identity_issuer_chain, + )), + ) +} + +async fn sev_ca_chain(State(state): State<Arc<MockCollateralState>>) -> impl IntoResponse { + binary(state.sev_snp.ca_chain_pem(), None) +} + +async fn sev_vcek(State(state): State<Arc<MockCollateralState>>) -> impl IntoResponse { + binary(state.sev_snp.vcek_der(), None) +} + +async fn tpm_root(State(state): State<Arc<MockCollateralState>>) -> impl IntoResponse { + binary(state.tpm.root_ca_der(), None) +} +async fn tpm_intermediate(State(state): State<Arc<MockCollateralState>>) -> impl IntoResponse { + binary(state.tpm.intermediate_der(), None) +} +async fn tpm_root_crl(State(state): State<Arc<MockCollateralState>>) -> impl IntoResponse { + binary(state.tpm.root_crl_der(), None) +} +async fn tpm_intermediate_crl(State(state): State<Arc<MockCollateralState>>) -> impl IntoResponse { + binary(state.tpm.intermediate_crl_der(), None) +} + +fn json_response(value: serde_json::Value, header: Option<(&str, String)>) -> Response<Body> { + let mut response = Response::new(Body::from(value.to_string())); + *response.status_mut() = StatusCode::OK; + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + if let Some((name, value)) = header { + let Ok(name) = axum::http::HeaderName::from_bytes(name.as_bytes()) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let Ok(value) = HeaderValue::from_str(&urlencoding::encode(&value)) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + response.headers_mut().insert(name, value); + } + response +} + +fn binary(value: Vec<u8>, header: Option<(&str, String)>) -> Response<Body> { + let mut response = Response::new(Body::from(value)); + *response.status_mut() = StatusCode::OK; + if let Some((name, value)) = header { + let Ok(name) = axum::http::HeaderName::from_bytes(name.as_bytes()) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let Ok(value) = HeaderValue::from_str(&urlencoding::encode(&value)) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + response.headers_mut().insert(name, value); + } + response +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn independent_host_and_guest_from_sys_config_seed_interoperate() { + let seed = [0x5a; 32]; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let url = format!("http://{addr}"); + // These states model separate host and CVM processes. No certificates, + // keys, or Arc state are exchanged after construction. + let host = Arc::new(MockCollateralState::from_seed(seed, &url).unwrap()); + let guest = MockCollateralState::from_seed(seed, &url).unwrap(); + let task = tokio::spawn(serve_listener(listener, host.clone())); + + let report_data = [0x61; 64]; + let tdx = guest.tdx.attest(report_data).unwrap(); + let collateral = dcap_qvl::collateral::CollateralClient::with_default_http(&url) + .unwrap() + .fetch(&tdx.quote) + .await + .unwrap(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let tdx_verifier = dcap_qvl::verify::QuoteVerifier::new(host.tdx.root_ca_der()); + tdx_verifier.verify(&tdx.quote, &collateral, now).unwrap(); + let mut tampered = tdx.quote.clone(); + tampered[200] ^= 1; + assert!(tdx_verifier.verify(&tampered, &collateral, now).is_err()); + + let sev = guest.sev_snp.attest(report_data).unwrap(); + let sev_verifier = sev_snp_qvl::QuoteVerifier::new_with_root( + sev_snp_qvl::AmdSnpProduct::Milan, + host.sev_snp.root_ca_pem().into_bytes(), + ); + sev_verifier + .verify(&sev.report, &sev.cert_chain, &report_data) + .unwrap(); + assert!(sev_verifier + .verify(&sev.report, &sev.cert_chain, &[0x62; 64]) + .is_err()); + + let qualifying = [0x63; 32]; + let tpm = guest.tpm.attest(&qualifying).unwrap(); + let tpm_verifier = tpm_qvl::QuoteVerifier::new(host.tpm.root_ca_pem()); + tpm_verifier.verify(&tpm, &host.tpm.collateral()).unwrap(); + assert!(crate::ensure_report_data(&qualifying, &[0x64; 32]).is_err()); + + let nsm = guest.nsm.attest(&report_data).unwrap(); + let verified = nsm_qvl::QuoteVerifier::new(host.nsm.root_ca_pem()) + .verify(&nsm, None, None) + .unwrap(); + assert_eq!(verified.user_data.as_deref(), Some(report_data.as_slice())); + + let wrong = MockCollateralState::from_seed([0xa5; 32], &url).unwrap(); + assert!(nsm_qvl::QuoteVerifier::new(wrong.nsm.root_ca_pem()) + .verify(&nsm, None, None) + .is_err()); + task.abort(); + } + + #[tokio::test] + async fn mock_pccs_and_kds_drive_real_qvls() { + let state = Arc::new(MockCollateralState::new().unwrap()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let task = tokio::spawn(serve_listener(listener, state.clone())); + let base = format!("http://{addr}"); + + let tdx_evidence = state.tdx.attest([0x42; 64]).unwrap(); + let collateral = dcap_qvl::collateral::CollateralClient::with_default_http(&base) + .unwrap() + .fetch(&tdx_evidence.quote) + .await + .unwrap(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + dcap_qvl::verify::QuoteVerifier::new(state.tdx.root_ca_der()) + .verify(&tdx_evidence.quote, &collateral, now) + .unwrap(); + + let sev_evidence = state.sev_snp.attest([0x43; 64]).unwrap(); + let kds = sev_snp_qvl::AmdKdsClient::with_base_url(format!("{base}/vcek/v1")).unwrap(); + sev_snp_qvl::QuoteVerifier::new_with_root( + sev_snp_qvl::AmdSnpProduct::Milan, + state.sev_snp.root_ca_pem().into_bytes(), + ) + .fetch_and_verify(&kds, &sev_evidence.report, &[], &[0x43; 64]) + .await + .unwrap(); + task.abort(); + } + + #[tokio::test] + async fn tdx_quote_collateral_and_tcb_matrix() { + let state = Arc::new(MockCollateralState::new().unwrap()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let base = format!("http://{addr}"); + let mut task = tokio::spawn(serve_listener(listener, state.clone())); + let evidence = state.tdx.attest([0x42; 64]).unwrap(); + let client = dcap_qvl::collateral::CollateralClient::with_default_http(&base).unwrap(); + let current = client.fetch(&evidence.quote).await.unwrap(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let verifier = dcap_qvl::verify::QuoteVerifier::new(state.tdx.root_ca_der()); + assert_eq!( + verifier + .verify(&evidence.quote, ¤t, now) + .unwrap() + .status, + "UpToDate" + ); + + let mut tampered_quote = evidence.quote.clone(); + tampered_quote[200] ^= 1; + assert!(verifier.verify(&tampered_quote, ¤t, now).is_err()); + + task.abort(); + let _ = task.await; + assert!(verifier.verify(&evidence.quote, ¤t, now).is_ok()); + assert!(client.fetch(&evidence.quote).await.is_err()); + + let listener = tokio::net::TcpListener::bind(addr).await.unwrap(); + task = tokio::spawn(serve_listener(listener, state.clone())); + let recovered = client.fetch(&evidence.quote).await.unwrap(); + assert!(verifier.verify(&evidence.quote, &recovered, now).is_ok()); + + let mut outdated = state + .tdx + .sample_collateral_with_tcb_status("OutOfDate") + .unwrap(); + outdated.pck_certificate_chain = recovered.pck_certificate_chain.clone(); + assert_eq!( + verifier + .verify(&evidence.quote, &outdated, now) + .unwrap() + .status, + "OutOfDate" + ); + + let mut revoked = state + .tdx + .sample_collateral_with_tcb_status("Revoked") + .unwrap(); + revoked.pck_certificate_chain = recovered.pck_certificate_chain.clone(); + assert!(verifier.verify(&evidence.quote, &revoked, now).is_err()); + + let expired_time = 4_200_000_000; + assert!(verifier + .verify(&evidence.quote, &recovered, expired_time) + .is_err()); + + let mut signature_invalid = recovered.clone(); + signature_invalid.tcb_info_signature[0] ^= 1; + assert!(verifier + .verify(&evidence.quote, &signature_invalid, now) + .is_err()); + + let mut malformed = recovered.clone(); + malformed.tcb_info = "{".into(); + assert!(verifier.verify(&evidence.quote, &malformed, now).is_err()); + task.abort(); + } + + #[tokio::test] + async fn tpm_aia_and_crl_service_builds_verifiable_collateral() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let state = + Arc::new(MockCollateralState::with_base_url(&format!("http://{addr}")).unwrap()); + let task = tokio::spawn(serve_listener(listener, state.clone())); + let quote = state.tpm.attest(&[0x42; 32]).unwrap(); + let root = state.tpm.root_ca_pem(); + let q = quote.clone(); + let collateral = tokio::task::spawn_blocking(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime + .block_on(tpm_qvl::get_collateral(&q, &root)) + .unwrap() + }) + .await + .unwrap(); + tpm_qvl::QuoteVerifier::new(state.tpm.root_ca_pem()) + .verify("e, &collateral) + .unwrap(); + task.abort(); + } +} diff --git a/dstack/crates/mock-attestation/src/sev_snp.rs b/dstack/crates/mock-attestation/src/sev_snp.rs new file mode 100644 index 000000000..720078198 --- /dev/null +++ b/dstack/crates/mock-attestation/src/sev_snp.rs @@ -0,0 +1,191 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Result; +use p384::ecdsa::{signature::DigestSigner, Signature as P384Signature, SigningKey}; +use p384::pkcs8::DecodePrivateKey; +use rcgen::{ + BasicConstraints, Certificate, CertificateParams, CertifiedKey, DnType, IsCa, KeyPair, + KeyUsagePurpose, +}; +use sev::firmware::guest::AttestationReport; +use sha2::{Digest, Sha384}; +use time::{Duration, OffsetDateTime}; + +pub struct SevSnpGenerator { + root: Certificate, + root_key: KeyPair, + ask: Certificate, + vcek: Certificate, + vcek_key: KeyPair, +} + +#[derive(serde::Serialize, serde::Deserialize)] +pub struct SevSnpEvidence { + pub report: Vec<u8>, + pub cert_chain: Vec<Vec<u8>>, +} + +impl SevSnpGenerator { + pub fn new() -> Result<Self> { + Self::from_seed(rand::random()) + } + + pub fn from_seed(seed: [u8; 32]) -> Result<Self> { + let CertifiedKey { + cert: root, + key_pair: root_key, + } = make_ca("Mock AMD Milan ARK", "sev-root", &seed, None)?; + let CertifiedKey { + cert: ask, + key_pair: ask_key, + } = make_ca( + "Mock AMD Milan ASK", + "sev-ask", + &seed, + Some((&root, &root_key)), + )?; + let vcek_key = crate::p384_key(&seed, "sev-vcek")?; + let mut params = base_params("Mock AMD VCEK")?; + params.key_usages.push(KeyUsagePurpose::DigitalSignature); + let vcek = params.signed_by(&vcek_key, &ask, &ask_key)?; + Ok(Self { + root, + root_key, + ask, + vcek, + vcek_key, + }) + } + + pub fn root_ca_pem(&self) -> String { + self.root.pem() + } + + pub fn root_key_pem(&self) -> String { + self.root_key.serialize_pem() + } + + pub fn ca_chain_pem(&self) -> Vec<u8> { + format!("{}{}", self.ask.pem(), self.root.pem()).into_bytes() + } + + pub fn vcek_der(&self) -> Vec<u8> { + self.vcek.der().to_vec() + } + + pub fn attest(&self, report_data: [u8; 64]) -> Result<SevSnpEvidence> { + self.attest_with_host_data(report_data, [0x22; 32]) + } + + pub fn attest_with_host_data( + &self, + report_data: [u8; 64], + host_data: [u8; 32], + ) -> Result<SevSnpEvidence> { + self.attest_with_measurement(report_data, host_data, [0x33; 48]) + } + + pub fn attest_with_measurement( + &self, + report_data: [u8; 64], + host_data: [u8; 32], + measurement: [u8; 48], + ) -> Result<SevSnpEvidence> { + let mut encoded = Vec::new(); + AttestationReport::default().write_bytes(&mut encoded)?; + encoded[0..4].copy_from_slice(&2u32.to_le_bytes()); + encoded[52..56].copy_from_slice(&1u32.to_le_bytes()); + encoded[0x50..0x90].copy_from_slice(&report_data); + encoded[0x90..0xc0].copy_from_slice(&measurement); + encoded[0xc0..0xe0].copy_from_slice(&host_data); + encoded[0x1a0..0x1e0].fill(0x33); + let signing_key = SigningKey::from_pkcs8_pem(&self.vcek_key.serialize_pem())?; + let signature: P384Signature = + signing_key.sign_digest(Sha384::new_with_prefix(&encoded[..0x2a0])); + write_amd_signature(&mut encoded[0x2a0..], &signature); + Ok(SevSnpEvidence { + report: encoded, + cert_chain: vec![self.ask.pem().into_bytes(), self.vcek.pem().into_bytes()], + }) + } +} + +fn make_ca( + name: &str, + label: &str, + seed: &[u8; 32], + issuer: Option<(&Certificate, &KeyPair)>, +) -> Result<CertifiedKey> { + let key_pair = crate::p384_key(seed, label)?; + let mut params = base_params(name)?; + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + params.key_usages.extend([ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + ]); + let cert = match issuer { + Some((issuer, issuer_key)) => params.signed_by(&key_pair, issuer, issuer_key)?, + None => params.self_signed(&key_pair)?, + }; + Ok(CertifiedKey { cert, key_pair }) +} + +fn base_params(name: &str) -> Result<CertificateParams> { + let mut params = CertificateParams::new(Vec::<String>::new())?; + params.distinguished_name.push(DnType::CommonName, name); + let now = OffsetDateTime::now_utc(); + params.not_before = now - Duration::days(1); + params.not_after = now + Duration::days(30); + Ok(params) +} + +fn write_amd_signature(output: &mut [u8], signature: &P384Signature) { + let bytes = signature.to_bytes(); + output[..144].fill(0); + for (dst, src) in output[..48].iter_mut().zip(bytes[..48].iter().rev()) { + *dst = *src; + } + for (dst, src) in output[72..120].iter_mut().zip(bytes[48..].iter().rev()) { + *dst = *src; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generated_report_passes_real_qvl_and_negative_cases_fail() { + let generator = SevSnpGenerator::new().unwrap(); + let report_data = [0x42; 64]; + let evidence = generator.attest(report_data).unwrap(); + let verifier = sev_snp_qvl::QuoteVerifier::new( + generator.root_ca_pem().into_bytes(), + generator.root_ca_pem().into_bytes(), + generator.root_ca_pem().into_bytes(), + ); + verifier + .verify(&evidence.report, &evidence.cert_chain, &report_data) + .unwrap(); + + let wrong = SevSnpGenerator::new().unwrap(); + let wrong_verifier = sev_snp_qvl::QuoteVerifier::new_with_root( + sev_snp_qvl::AmdSnpProduct::Milan, + wrong.root_ca_pem().into_bytes(), + ); + assert!(wrong_verifier + .verify(&evidence.report, &evidence.cert_chain, &report_data) + .is_err()); + let mut tampered = evidence.report.clone(); + tampered[0x100] ^= 1; + assert!(verifier + .verify(&tampered, &evidence.cert_chain, &report_data) + .is_err()); + assert!(verifier + .verify(&evidence.report, &evidence.cert_chain, &[0x24; 64]) + .is_err()); + } +} diff --git a/dstack/crates/mock-attestation/src/tdx.rs b/dstack/crates/mock-attestation/src/tdx.rs new file mode 100644 index 000000000..821ea4e70 --- /dev/null +++ b/dstack/crates/mock-attestation/src/tdx.rs @@ -0,0 +1,552 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Result; +use dcap_qvl::quote::{ + AuthData, AuthDataV4, CertificationData, Data, EnclaveReport, Header, + QEReportCertificationData, Quote, Report, TDReport10, +}; +use dcap_qvl::QuoteCollateralV3; +use p256::ecdsa::{signature::Signer, Signature, SigningKey}; +use p256::pkcs8::{DecodePrivateKey, EncodePrivateKey}; +use rcgen::{ + BasicConstraints, Certificate, CertificateParams, CertificateRevocationListParams, + CertifiedKey, CustomExtension, DnType, ExtendedKeyUsagePurpose, IsCa, KeyIdMethod, KeyPair, + KeyUsagePurpose, RemoteKeyPair, SerialNumber, SignatureAlgorithm, PKCS_ECDSA_P256_SHA256, +}; +use scale::Encode; +use serde_json::json; +use sha2::{Digest, Sha256}; +use time::OffsetDateTime; + +const MOCK_PKI_NOT_BEFORE: i64 = 1_577_836_800; // 2020-01-01T00:00:00Z +const MOCK_PKI_NOT_AFTER: i64 = 4_102_444_800; // 2100-01-01T00:00:00Z + +/// Collateral validity, matching the mock PKI window above so a seeded +/// generator's output is fully deterministic. A test asserts the two stay in +/// sync. +const COLLATERAL_ISSUE_DATE: &str = "2020-01-01T00:00:00Z"; +const COLLATERAL_NEXT_UPDATE: &str = "2100-01-01T00:00:00Z"; + +const INTEL_QE_VENDOR_ID: [u8; 16] = [ + 0x93, 0x9a, 0x72, 0x33, 0xf7, 0x9c, 0x4c, 0xa9, 0x94, 0x0a, 0x0d, 0xb3, 0x95, 0x7f, 0x06, 0x07, +]; + +pub struct TdxGenerator { + root: Certificate, + root_signing_key: SigningKey, + pck_ca: Certificate, + pck: Certificate, + pck_key: SigningKey, + pck_crl: Vec<u8>, + tcb_signer: Certificate, + tcb_signer_key: SigningKey, + qe_signer: Certificate, + qe_signer_key: SigningKey, + root_crl: Vec<u8>, +} + +struct DeterministicP256KeyPair { + key: SigningKey, + public_key: Vec<u8>, +} + +impl DeterministicP256KeyPair { + fn new(key: SigningKey) -> Self { + let public_key = key + .verifying_key() + .to_encoded_point(false) + .as_bytes() + .to_vec(); + Self { key, public_key } + } +} + +impl RemoteKeyPair for DeterministicP256KeyPair { + fn public_key(&self) -> &[u8] { + &self.public_key + } + + fn sign(&self, message: &[u8]) -> Result<Vec<u8>, rcgen::Error> { + let signature: Signature = self.key.sign(message); + Ok(signature.to_der().as_bytes().to_vec()) + } + + fn algorithm(&self) -> &'static SignatureAlgorithm { + &PKCS_ECDSA_P256_SHA256 + } +} + +pub struct TdxEvidence { + pub quote: Vec<u8>, + pub collateral: QuoteCollateralV3, +} + +impl TdxGenerator { + pub fn new() -> Result<Self> { + Self::from_seed(rand::random()) + } + + pub fn from_seed(seed: [u8; 32]) -> Result<Self> { + let ( + CertifiedKey { + cert: root, + key_pair: root_key, + }, + root_signing_key, + ) = make_root(&seed)?; + let (pck_ca, pck_ca_key) = make_ca( + "Mock Intel SGX PCK Platform CA", + "tdx-pck-ca", + &seed, + &root, + &root_key, + )?; + let (pck, pck_key) = make_leaf( + "Mock Intel SGX PCK Certificate", + "tdx-pck", + &seed, + &pck_ca, + &pck_ca_key, + true, + )?; + let (tcb_signer, tcb_signer_key) = make_leaf( + "Mock Intel SGX TCB Signing", + "tdx-tcb", + &seed, + &root, + &root_key, + false, + )?; + let (qe_signer, qe_signer_key) = make_leaf( + "Mock Intel SGX QE Identity Signing", + "tdx-qe", + &seed, + &root, + &root_key, + false, + )?; + let pck_crl = CertificateRevocationListParams { + this_update: fixed_time(MOCK_PKI_NOT_BEFORE)?, + next_update: fixed_time(MOCK_PKI_NOT_AFTER)?, + crl_number: SerialNumber::from(1u64), + issuing_distribution_point: None, + revoked_certs: Vec::new(), + key_identifier_method: KeyIdMethod::Sha256, + } + .signed_by(&pck_ca, &pck_ca_key)? + .der() + .to_vec(); + let root_crl = CertificateRevocationListParams { + this_update: fixed_time(MOCK_PKI_NOT_BEFORE)?, + next_update: fixed_time(MOCK_PKI_NOT_AFTER)?, + crl_number: SerialNumber::from(1u64), + issuing_distribution_point: None, + revoked_certs: Vec::new(), + key_identifier_method: KeyIdMethod::Sha256, + } + .signed_by(&root, &root_key)? + .der() + .to_vec(); + Ok(Self { + root, + root_signing_key, + pck_ca, + pck, + pck_key, + pck_crl, + tcb_signer, + tcb_signer_key, + qe_signer, + qe_signer_key, + root_crl, + }) + } + + pub fn root_ca_der(&self) -> Vec<u8> { + self.root.der().to_vec() + } + pub fn root_ca_pem(&self) -> String { + self.root.pem() + } + pub fn root_key_pem(&self) -> Result<String> { + Ok(self + .root_signing_key + .to_pkcs8_pem(Default::default())? + .to_string()) + } + + pub fn sample_collateral(&self) -> Result<QuoteCollateralV3> { + self.collateral() + } + + #[cfg(test)] + pub(crate) fn sample_collateral_with_tcb_status( + &self, + tcb_status: &str, + ) -> Result<QuoteCollateralV3> { + self.collateral_with_tcb_status(tcb_status) + } + + pub fn root_crl_der(&self) -> Vec<u8> { + self.root_crl.clone() + } + + pub fn pck_crl_der(&self) -> Vec<u8> { + self.pck_crl.clone() + } + + pub fn attest(&self, report_data: [u8; 64]) -> Result<TdxEvidence> { + self.attest_with_rtmrs( + report_data, + [[0x20; 48], [0x21; 48], [0x22; 48], [0x23; 48]], + ) + } + + pub fn attest_with_rtmrs( + &self, + report_data: [u8; 64], + rtmrs: [[u8; 48]; 4], + ) -> Result<TdxEvidence> { + self.attest_with_measurements(report_data, [0x11; 48], rtmrs) + } + + pub fn attest_with_measurements( + &self, + report_data: [u8; 64], + mrtd: [u8; 48], + rtmrs: [[u8; 48]; 4], + ) -> Result<TdxEvidence> { + let auth_key = SigningKey::random(&mut rand::thread_rng()); + let auth_pub = auth_key.verifying_key().to_encoded_point(false); + let auth_pub: [u8; 64] = auth_pub.as_bytes()[1..] + .try_into() + .map_err(|_| anyhow::anyhow!("invalid P-256 public key length"))?; + let qe_auth = vec![0u8; 32]; + let mut qe_hash_input = Vec::from(auth_pub); + qe_hash_input.extend_from_slice(&qe_auth); + let mut qe_report = EnclaveReport { + cpu_svn: [0; 16], + misc_select: 0, + reserved1: [0; 28], + attributes: [0; 16], + mr_enclave: [0; 32], + reserved2: [0; 32], + mr_signer: [0x44; 32], + reserved3: [0; 96], + isv_prod_id: 1, + isv_svn: 1, + reserved4: [0; 60], + report_data: [0; 64], + }; + qe_report.report_data[..32].copy_from_slice(&Sha256::digest(qe_hash_input)); + let qe_report_bytes: [u8; 384] = qe_report + .encode() + .try_into() + .map_err(|bytes: Vec<u8>| anyhow::anyhow!("invalid QE report size {}", bytes.len()))?; + let qe_sig: Signature = self.pck_key.sign(&qe_report_bytes); + + let pck_chain = + format!("{}{}{}", self.pck.pem(), self.pck_ca.pem(), self.root.pem()).into_bytes(); + let qe_certification = QEReportCertificationData { + qe_report: qe_report_bytes, + qe_report_signature: qe_sig.to_bytes().into(), + qe_auth_data: Data::new(qe_auth), + certification_data: CertificationData { + cert_type: 5, + body: Data::new(pck_chain), + }, + }; + let td_report = TDReport10 { + tee_tcb_svn: [0; 16], + mr_seam: [0; 48], + mr_signer_seam: [0; 48], + seam_attributes: [0; 8], + td_attributes: [0, 0, 0, 0x10, 0, 0, 0, 0], + xfam: [0; 8], + mr_td: mrtd, + mr_config_id: [0; 48], + mr_owner: [0; 48], + mr_owner_config: [0; 48], + rt_mr0: rtmrs[0], + rt_mr1: rtmrs[1], + rt_mr2: rtmrs[2], + rt_mr3: rtmrs[3], + report_data, + }; + let header = Header { + version: 4, + attestation_key_type: 2, + tee_type: 0x81, + qe_svn: 1, + pce_svn: 0, + qe_vendor_id: INTEL_QE_VENDOR_ID, + user_data: [0; 20], + }; + let mut auth = AuthDataV4 { + ecdsa_signature: [0; 64], + ecdsa_attestation_key: auth_pub, + certification_data: CertificationData { + cert_type: 6, + body: Data::new(qe_certification.encode()), + }, + qe_report_data: qe_certification, + }; + let mut quote = Quote { + header, + report: Report::TD10(td_report), + auth_data: AuthData::V4(auth.clone()), + }; + let raw = quote.encode(); + let quote_sig: Signature = auth_key.sign(&raw[..quote.signed_length()]); + auth.ecdsa_signature = quote_sig.to_bytes().into(); + quote.auth_data = AuthData::V4(auth); + + Ok(TdxEvidence { + quote: quote.encode(), + collateral: self.collateral()?, + }) + } + + fn collateral(&self) -> Result<QuoteCollateralV3> { + self.collateral_with_tcb_status("UpToDate") + } + + fn collateral_with_tcb_status(&self, tcb_status: &str) -> Result<QuoteCollateralV3> { + // The collateral must be as deterministic as the certificates: two + // generators built from the same seed have to produce byte-identical + // collateral — same JSON, and therefore same signatures — no matter + // when or in which process they run. Wall-clock stamps here made that + // equality depend on both generations landing in the same second. + // Use the mock PKI's fixed validity window instead; dcap-qvl only + // requires issueDate <= now <= nextUpdate. + let issue = COLLATERAL_ISSUE_DATE; + let next = COLLATERAL_NEXT_UPDATE; + let tcb_info = json!({ + "id":"TDX", "version":3, "issueDate":issue, "nextUpdate":next, + "fmspc":"000000000000", "pceId":"0000", "tcbType":0, "tcbEvaluationDataNumber":1, + "tdxModule":{"mrsigner":"00".repeat(48),"attributes":"00".repeat(8),"attributesMask":"00".repeat(8)}, + "tcbLevels":[{"tcb":{"sgxtcbcomponents":vec![json!({"svn":0});16],"pcesvn":0,"tdxtcbcomponents":vec![json!({"svn":0});16]},"tcbDate":issue,"tcbStatus":tcb_status}] + }).to_string(); + let qe_identity = json!({ + "id":"TD_QE", "version":2, "issueDate":issue, "nextUpdate":next, + "tcbEvaluationDataNumber":1, "miscselect":"00000000", "miscselectMask":"00000000", + "attributes":"00".repeat(16), "attributesMask":"00".repeat(16), "mrsigner":"44".repeat(32), + "isvprodid":1, "tcbLevels":[{"tcb":{"isvsvn":1},"tcbDate":issue,"tcbStatus":"UpToDate"}] + }).to_string(); + Ok(QuoteCollateralV3 { + pck_crl_issuer_chain: format!("{}{}", self.pck_ca.pem(), self.root.pem()), + root_ca_crl: self.root_crl.clone(), + pck_crl: self.pck_crl.clone(), + tcb_info_issuer_chain: format!("{}{}", self.tcb_signer.pem(), self.root.pem()), + tcb_info_signature: sign_raw(&self.tcb_signer_key, tcb_info.as_bytes())?, + tcb_info, + qe_identity_issuer_chain: format!("{}{}", self.qe_signer.pem(), self.root.pem()), + qe_identity_signature: sign_raw(&self.qe_signer_key, qe_identity.as_bytes())?, + qe_identity, + pck_certificate_chain: None, + }) + } +} + +fn deterministic_key_pair(seed: &[u8; 32], label: &str) -> Result<(KeyPair, SigningKey)> { + let serialized = crate::p256_key(seed, label)?; + let signing_key = SigningKey::from_pkcs8_pem(&serialized.serialize_pem())?; + let key_pair = + KeyPair::from_remote(Box::new(DeterministicP256KeyPair::new(signing_key.clone())))?; + Ok((key_pair, signing_key)) +} + +fn make_root(seed: &[u8; 32]) -> Result<(CertifiedKey, SigningKey)> { + let (key_pair, signing_key) = deterministic_key_pair(seed, "tdx-root")?; + let mut params = cert_params("Mock Intel SGX Root CA")?; + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + params.key_usages.extend([ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + ]); + let cert = params.self_signed(&key_pair)?; + Ok((CertifiedKey { cert, key_pair }, signing_key)) +} + +fn make_ca( + name: &str, + label: &str, + seed: &[u8; 32], + issuer: &Certificate, + issuer_key: &KeyPair, +) -> Result<(Certificate, KeyPair)> { + let (key, _) = deterministic_key_pair(seed, label)?; + let mut params = cert_params(name)?; + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + params.key_usages.extend([ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + ]); + let cert = params.signed_by(&key, issuer, issuer_key)?; + Ok((cert, key)) +} + +fn make_leaf( + name: &str, + label: &str, + seed: &[u8; 32], + root: &Certificate, + root_key: &KeyPair, + pck: bool, +) -> Result<(Certificate, SigningKey)> { + let (key, signing_key) = deterministic_key_pair(seed, label)?; + let mut params = cert_params(name)?; + params.key_usages.push(KeyUsagePurpose::DigitalSignature); + params + .extended_key_usages + .push(ExtendedKeyUsagePurpose::ServerAuth); + if pck { + params.custom_extensions.push(pck_extension()); + } + let cert = params.signed_by(&key, root, root_key)?; + Ok((cert, signing_key)) +} + +fn cert_params(name: &str) -> Result<CertificateParams> { + let mut params = CertificateParams::new(vec!["mock.dstack.invalid".into()])?; + params.distinguished_name.push(DnType::CommonName, name); + params.serial_number = Some(SerialNumber::from(42u64)); + params.not_before = fixed_time(MOCK_PKI_NOT_BEFORE)?; + params.not_after = fixed_time(MOCK_PKI_NOT_AFTER)?; + Ok(params) +} + +fn fixed_time(timestamp: i64) -> Result<OffsetDateTime> { + Ok(OffsetDateTime::from_unix_timestamp(timestamp)?) +} + +fn pck_extension() -> CustomExtension { + fn oid(writer: yasna::DERWriter, oid: &[u64]) { + writer.write_oid(&yasna::models::ObjectIdentifier::from_slice(oid)); + } + let der = yasna::construct_der(|writer| { + writer.write_sequence(|writer| { + let entries: Vec<(&[u64], Vec<u8>, u8)> = vec![ + (&[1, 2, 840, 113741, 1, 13, 1, 1], vec![0; 16], 0), + (&[1, 2, 840, 113741, 1, 13, 1, 3], vec![0; 2], 0), + (&[1, 2, 840, 113741, 1, 13, 1, 4], vec![0; 6], 0), + (&[1, 2, 840, 113741, 1, 13, 1, 5], vec![0], 2), + ]; + for (entry_oid, value, kind) in entries { + writer.next().write_sequence(|w| { + oid(w.next(), entry_oid); + if kind == 2 { + w.next().write_enum(0) + } else { + w.next().write_bytes(&value) + } + }); + } + writer.next().write_sequence(|w| { + oid(w.next(), &[1, 2, 840, 113741, 1, 13, 1, 2]); + w.next().write_sequence(|w| { + w.next().write_sequence(|w| { + oid(w.next(), &[1, 2, 840, 113741, 1, 13, 1, 2, 17]); + w.next().write_u8(0); + }); + w.next().write_sequence(|w| { + oid(w.next(), &[1, 2, 840, 113741, 1, 13, 1, 2, 18]); + w.next().write_bytes(&[0; 16]); + }); + }); + }); + }) + }); + CustomExtension::from_oid_content(&[1, 2, 840, 113741, 1, 13, 1], der) +} + +fn sign_raw(key: &SigningKey, message: &[u8]) -> Result<Vec<u8>> { + let sig: Signature = key.sign(message); + Ok(sig.to_bytes().to_vec()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The collateral advertises the same validity window the mock PKI + /// certificates carry; if one side moves, this pins the other to follow. + #[test] + fn collateral_validity_matches_the_mock_pki_window() { + let issue = chrono::DateTime::parse_from_rfc3339(COLLATERAL_ISSUE_DATE).unwrap(); + let next = chrono::DateTime::parse_from_rfc3339(COLLATERAL_NEXT_UPDATE).unwrap(); + assert_eq!(issue.timestamp(), MOCK_PKI_NOT_BEFORE); + assert_eq!(next.timestamp(), MOCK_PKI_NOT_AFTER); + } + + /// The collateral must not embed the wall clock: two generations from + /// the same seed have to be byte-identical even when they straddle a + /// second boundary — that was a real CI flake in + /// `seeded_hierarchies_are_cross_process_compatible`, which only holds + /// per-run. Asserting the fixed stamps guards against `now()` creeping + /// back in. + #[test] + fn collateral_timestamps_are_fixed_not_wall_clock() { + let generator = TdxGenerator::from_seed([0x31; 32]).unwrap(); + let collateral = generator.sample_collateral().unwrap(); + for doc in [&collateral.tcb_info, &collateral.qe_identity] { + assert!(doc.contains("\"issueDate\":\"2020-01-01T00:00:00Z\"")); + assert!(doc.contains("\"nextUpdate\":\"2100-01-01T00:00:00Z\"")); + } + } + + #[test] + fn seeded_hierarchies_are_cross_process_compatible() { + let first = TdxGenerator::from_seed([0x31; 32]).unwrap(); + let second = TdxGenerator::from_seed([0x31; 32]).unwrap(); + assert_eq!(first.root_ca_der(), second.root_ca_der()); + assert_eq!(first.root_crl_der(), second.root_crl_der()); + + let evidence = first.attest([0x42; 64]).unwrap(); + let collateral = second.sample_collateral().unwrap(); + assert_eq!(evidence.collateral, collateral); + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + dcap_qvl::verify::QuoteVerifier::new(second.root_ca_der()) + .verify(&evidence.quote, &collateral, now) + .unwrap(); + } + + #[test] + fn generated_quote_passes_real_qvl_and_negative_cases_fail() { + let generator = TdxGenerator::new().unwrap(); + let report_data = [0x42; 64]; + let evidence = generator.attest(report_data).unwrap(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let verifier = dcap_qvl::verify::QuoteVerifier::new(generator.root_ca_der()); + let verified = verifier + .verify(&evidence.quote, &evidence.collateral, now) + .unwrap(); + assert_eq!(verified.report.as_td10().unwrap().report_data, report_data); + assert!( + dcap_qvl::verify::QuoteVerifier::new(TdxGenerator::new().unwrap().root_ca_der()) + .verify(&evidence.quote, &evidence.collateral, now) + .is_err() + ); + let mut tampered = evidence.quote.clone(); + tampered[100] ^= 1; + assert!(verifier + .verify(&tampered, &evidence.collateral, now) + .is_err()); + assert!(crate::ensure_report_data( + &verified.report.as_td10().unwrap().report_data, + &[0x24; 64] + ) + .is_err()); + } +} diff --git a/dstack/crates/mock-attestation/src/tpm.rs b/dstack/crates/mock-attestation/src/tpm.rs new file mode 100644 index 000000000..78b038dec --- /dev/null +++ b/dstack/crates/mock-attestation/src/tpm.rs @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Result; +use dstack_types::Platform; +use p256::ecdsa::{signature::hazmat::PrehashSigner, Signature, SigningKey}; +use p256::pkcs8::DecodePrivateKey; +use rcgen::{ + BasicConstraints, Certificate, CertificateParams, CertificateRevocationListParams, + CrlDistributionPoint, CustomExtension, DnType, IsCa, KeyIdMethod, KeyPair, KeyUsagePurpose, + SerialNumber, +}; +use sha2::{Digest, Sha256}; +use time::{Duration, OffsetDateTime}; +use tpm_qvl::QuoteCollateral; +use tpm_types::{PcrValue, TpmQuote}; + +pub struct TpmGenerator { + root: Certificate, + root_key: KeyPair, + intermediate: Certificate, + ak: Certificate, + ak_key: KeyPair, + intermediate_crl: Vec<u8>, + root_crl: Vec<u8>, +} + +impl TpmGenerator { + pub fn new() -> Result<Self> { + Self::with_base_url("http://127.0.0.1:8088") + } + + pub fn with_base_url(base_url: &str) -> Result<Self> { + Self::from_seed(rand::random(), base_url) + } + + pub fn from_seed(seed: [u8; 32], base_url: &str) -> Result<Self> { + let root_key = crate::p256_key(&seed, "tpm-root")?; + let mut root_params = params("Mock TPM Root CA")?; + root_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + root_params.key_usages.extend([ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + ]); + root_params + .crl_distribution_points + .push(CrlDistributionPoint { + uris: vec![format!("{base_url}/tpm/crl/root.crl")], + }); + let root = root_params.self_signed(&root_key)?; + + let intermediate_key = crate::p256_key(&seed, "tpm-intermediate")?; + let mut intermediate_params = params("Mock TPM Attestation Intermediate CA")?; + intermediate_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + intermediate_params.key_usages.extend([ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + ]); + intermediate_params + .custom_extensions + .push(aia(&format!("{base_url}/tpm/aia/root.pem"))); + intermediate_params + .crl_distribution_points + .push(CrlDistributionPoint { + uris: vec![format!("{base_url}/tpm/crl/root.crl")], + }); + let intermediate = intermediate_params.signed_by(&intermediate_key, &root, &root_key)?; + let ak_key = crate::p256_key(&seed, "tpm-ak")?; + let mut ak_params = params("Mock TPM Attestation Key")?; + ak_params.key_usages.push(KeyUsagePurpose::DigitalSignature); + ak_params + .custom_extensions + .push(aia(&format!("{base_url}/tpm/aia/intermediate.der"))); + ak_params + .crl_distribution_points + .push(CrlDistributionPoint { + uris: vec![format!("{base_url}/tpm/crl/intermediate.crl")], + }); + let ak = ak_params.signed_by(&ak_key, &intermediate, &intermediate_key)?; + let now = OffsetDateTime::now_utc(); + let crl = |issuer: &Certificate, key: &KeyPair, number| -> Result<Vec<u8>> { + Ok(CertificateRevocationListParams { + this_update: now - Duration::days(1), + next_update: now + Duration::days(30), + crl_number: SerialNumber::from(number), + issuing_distribution_point: None, + revoked_certs: Vec::new(), + key_identifier_method: KeyIdMethod::Sha256, + } + .signed_by(issuer, key)? + .der() + .to_vec()) + }; + let root_crl = crl(&root, &root_key, 1)?; + let intermediate_crl = crl(&intermediate, &intermediate_key, 2)?; + Ok(Self { + root, + root_key, + intermediate, + ak, + ak_key, + intermediate_crl, + root_crl, + }) + } + + pub fn root_ca_pem(&self) -> String { + self.root.pem() + } + + pub fn root_ca_der(&self) -> Vec<u8> { + self.root.der().to_vec() + } + + pub fn root_key_pem(&self) -> String { + self.root_key.serialize_pem() + } + + pub fn collateral(&self) -> QuoteCollateral { + QuoteCollateral { + cert_chain_pem: self.intermediate.pem(), + crls: vec![self.intermediate_crl.clone()], + root_ca_crl: Some(self.root_crl.clone()), + } + } + + pub fn intermediate_der(&self) -> Vec<u8> { + self.intermediate.der().to_vec() + } + pub fn intermediate_crl_der(&self) -> Vec<u8> { + self.intermediate_crl.clone() + } + pub fn root_crl_der(&self) -> Vec<u8> { + self.root_crl.clone() + } + + pub fn attest(&self, qualifying_data: &[u8]) -> Result<TpmQuote> { + let pcr = PcrValue { + index: 14, + algorithm: "sha256".into(), + value: vec![0x11; 32], + }; + let pcr_digest = Sha256::digest(&pcr.value); + let mut message = Vec::new(); + message.extend_from_slice(&0xff54_4347u32.to_be_bytes()); + message.extend_from_slice(&0x8018u16.to_be_bytes()); + message.extend_from_slice(&0u16.to_be_bytes()); // qualified signer + message.extend_from_slice(&(qualifying_data.len() as u16).to_be_bytes()); + message.extend_from_slice(qualifying_data); + message.extend_from_slice(&0u64.to_be_bytes()); // clock + message.extend_from_slice(&0u32.to_be_bytes()); // reset count + message.extend_from_slice(&0u32.to_be_bytes()); // restart count + message.push(1); // safe + message.extend_from_slice(&0u64.to_be_bytes()); // firmware + message.extend_from_slice(&1u32.to_be_bytes()); // selection count + message.extend_from_slice(&0x000bu16.to_be_bytes()); + message.push(3); + message.extend_from_slice(&[0, 0x40, 0]); // PCR14 + message.extend_from_slice(&(pcr_digest.len() as u16).to_be_bytes()); + message.extend_from_slice(&pcr_digest); + + let digest = Sha256::digest(&message); + let key = SigningKey::from_pkcs8_pem(&self.ak_key.serialize_pem())?; + let signature: Signature = key.sign_prehash(&digest)?; + let bytes = signature.to_bytes(); + let mut tpm_signature = Vec::with_capacity(72); + tpm_signature.extend_from_slice(&0x0018u16.to_be_bytes()); + tpm_signature.extend_from_slice(&0x000bu16.to_be_bytes()); + tpm_signature.extend_from_slice(&32u16.to_be_bytes()); + tpm_signature.extend_from_slice(&bytes[..32]); + tpm_signature.extend_from_slice(&32u16.to_be_bytes()); + tpm_signature.extend_from_slice(&bytes[32..]); + + Ok(TpmQuote { + message, + signature: tpm_signature, + pcr_values: vec![pcr], + ak_cert: self.ak.der().to_vec(), + platform: Platform::Gcp, + event_log: Vec::new(), + }) + } +} + +fn aia(url: &str) -> CustomExtension { + let der = yasna::construct_der(|writer| { + writer.write_sequence(|writer| { + writer.next().write_sequence(|writer| { + writer + .next() + .write_oid(&yasna::models::ObjectIdentifier::from_slice(&[ + 1, 3, 6, 1, 5, 5, 7, 48, 2, + ])); + writer + .next() + .write_tagged_implicit(yasna::Tag::context(6), |writer| { + writer.write_ia5_string(url) + }); + }); + }) + }); + CustomExtension::from_oid_content(&[1, 3, 6, 1, 5, 5, 7, 1, 1], der) +} + +fn params(name: &str) -> Result<CertificateParams> { + let mut params = CertificateParams::new(Vec::<String>::new())?; + params.distinguished_name.push(DnType::CommonName, name); + let now = OffsetDateTime::now_utc(); + params.not_before = now - Duration::days(1); + params.not_after = now + Duration::days(30); + Ok(params) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generated_quote_passes_real_qvl_and_negative_cases_fail() { + let generator = TpmGenerator::new().unwrap(); + let qualifying_data = [0x42; 32]; + let quote = generator.attest(&qualifying_data).unwrap(); + let verifier = tpm_qvl::QuoteVerifier::new(generator.root_ca_pem()); + verifier.verify("e, &generator.collateral()).unwrap(); + + let wrong = TpmGenerator::new().unwrap(); + assert!(tpm_qvl::QuoteVerifier::new(wrong.root_ca_pem()) + .verify("e, &generator.collateral()) + .is_err()); + let mut tampered = quote.clone(); + tampered.message[10] ^= 1; + assert!(verifier.verify(&tampered, &generator.collateral()).is_err()); + let verified = verifier.verify("e, &generator.collateral()).unwrap(); + assert!(crate::ensure_report_data(&verified.attest.qualified_data, &[0x24; 32]).is_err()); + } +} diff --git a/dstack/crates/qemu-acpi/Cargo.toml b/dstack/crates/qemu-acpi/Cargo.toml new file mode 100644 index 000000000..9ca99aedf --- /dev/null +++ b/dstack/crates/qemu-acpi/Cargo.toml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "qemu-acpi" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +description = "Pure Rust generator for QEMU-compatible ACPI measurement blobs" + +[dependencies] +acpi_tables.workspace = true +thiserror.workspace = true + +[dev-dependencies] +hex = { workspace = true, features = ["alloc"] } +sha2.workspace = true diff --git a/dstack/crates/qemu-acpi/examples/dump.rs b/dstack/crates/qemu-acpi/examples/dump.rs new file mode 100644 index 000000000..1645447ce --- /dev/null +++ b/dstack/crates/qemu-acpi/examples/dump.rs @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// SPDX-License-Identifier: Apache-2.0 + +use std::error::Error; +use std::str::FromStr; + +use qemu_acpi::{build, MachineConfig, QemuVersion}; + +fn optional<T: FromStr>(args: &mut impl Iterator<Item = String>, default: T) -> Result<T, String> { + match args.next() { + Some(value) => value.parse().map_err(|_| format!("invalid value: {value}")), + None => Ok(default), + } +} + +fn main() -> Result<(), Box<dyn Error>> { + let mut args = std::env::args().skip(1); + let nics: u32 = optional(&mut args, 1)?; + let cpus: u32 = optional(&mut args, 1)?; + let version = QemuVersion::from_str(&optional::<String>(&mut args, "11.1.0".into())?) + .map_err(|error| format!("invalid QEMU version: {error}"))?; + let gpus = optional(&mut args, 0)?; + let nvswitches = optional(&mut args, 0)?; + let hugepages = optional::<u8>(&mut args, 0)? == 1; + let root_verity = optional::<u8>(&mut args, 1)? != 0; + let hotplug_off = optional::<u8>(&mut args, 0)? == 1; + let smm = optional::<u8>(&mut args, 0)? == 1; + let hole = optional::<u64>(&mut args, 0)?; + let memory_size = optional(&mut args, 2u64 << 30)?; + let volumes = optional(&mut args, 0)?; + let pic = optional::<u8>(&mut args, 0)? == 1; + let blobs = build(&MachineConfig { + qemu_version: version, + cpu_count: cpus, + memory_size, + pic, + smm, + hugepages, + num_gpus: gpus, + num_nvswitches: nvswitches, + num_nics: nics, + num_verity_volumes: volumes, + hotplug_off, + root_verity, + pci_hole64_size: (hole != 0).then_some(hole), + })?; + let output_dir = std::env::var_os("QEMU_ACPI_OUTPUT_DIR") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| std::path::PathBuf::from("/tmp")); + std::fs::create_dir_all(&output_dir)?; + std::fs::write(output_dir.join("tables.bin"), blobs.tables)?; + std::fs::write(output_dir.join("loader.bin"), blobs.loader)?; + std::fs::write(output_dir.join("rsdp.bin"), blobs.rsdp)?; + Ok(()) +} diff --git a/dstack/crates/qemu-acpi/fixtures/README.md b/dstack/crates/qemu-acpi/fixtures/README.md new file mode 100644 index 000000000..109eb4bd8 --- /dev/null +++ b/dstack/crates/qemu-acpi/fixtures/README.md @@ -0,0 +1,107 @@ +# QEMU ACPI compatibility fixtures + +These binary fixtures are machine-generated `etc/acpi/tables` blobs. They +capture QEMU's externally observable ACPI ABI and contain no QEMU source code. +They must never be regenerated from the Rust implementation. + +## Pinned production reference + +The fixtures in this directory were generated from: + +- repository: <https://github.com/kvinwang/qemu-tdx> +- branch: `dstack-qemu-acpi-11.1-compat` +- revision: `9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195` +- dstack image inputs: `dstack-0.5.5/ovmf.fd` and + `dstack-0.5.5/bzImage` + +Build the reference in a clean build directory: + +```bash +git clone https://github.com/kvinwang/qemu-tdx.git qemu-tdx +cd qemu-tdx +git checkout 9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195 +git apply /path/to/qemu-acpi/scripts/qemu-dump-all-blobs.patch +mkdir build-acpi && cd build-acpi +CFLAGS='-DDUMP_ACPI_TABLES -Wno-builtin-macro-redefined -D__DATE__="" -D__TIME__="" -D__TIMESTAMP__=""' \ +LDFLAGS='-Wl,--build-id=none' \ +../configure --prefix=/tmp/qemu-acpi-compat-install \ + --target-list=x86_64-softmmu --disable-werror +ninja qemu-system-x86_64 +``` + +Run the complete three-blob differential matrix with: + +```bash +../scripts/differential-qemu.sh \ + /path/to/qemu-tdx/build-acpi/qemu-system-x86_64 \ + /path/to/dstack-0.5.5 +``` + +The script compares `etc/acpi/tables`, `etc/table-loader`, and +`etc/acpi/rsdp` byte for byte for normal, NUMA, and NUMA/PXB configurations. +The dump patch is test-only instrumentation and does not change table +construction. + +The trimmed `*-base.bin` files are test-only byte oracles. They are never read +by non-test code and are not templates for generation. QEMU's trailing zero +allocation is removed at the first all-zero table signature. The four variants +for each compatibility family cover the Cartesian product of: + +- CPU hotplug enabled or disabled; and +- ordinary Q35 or NUMA with one PXB. + +`qemu-11.1-q35-one-nic.bin` is an untrimmed byte-for-byte regression fixture. +Fixture names identify the emulated compatibility version, not necessarily the +source tree's release number. + +## Compatibility-reference scope + +The pinned reference is based on QEMU 9.2.1. Its 10.x and 11.x profiles are a +dstack-maintained compatibility implementation composed from the relevant +upstream ACPI changes. Therefore matching this reference proves compatibility +with the QEMU binary used by dstack production, not by itself with every +unmodified upstream QEMU release. Genuine upstream QEMU cross-validation must +be recorded separately whenever a new compatibility family is added. + +## Genuine upstream cross-validation + +The following unmodified upstream release revisions were built locally, then +instrumented with `scripts/qemu-upstream-dump.patch`: + +- QEMU 10.2.4: `3e0bcba1ca7d6607ca49a988d165f052a3a53323` +- QEMU 11.0.3: `aeec49e8170de7846f476124602cf7acd400c3df` + +For each release, `scripts/differential-upstream.sh` compared all three blobs +for normal, NUMA, and NUMA/PXB topologies. All six cases matched byte for byte. +The upstream instrumentation normalizes only values that the production +`DUMP_ACPI_TABLES` patch also normalizes (PM I/O, PCI windows, and MCFG), and +bypasses TDX realization so the comparison can run without a TDX-capable KVM. +It does not backport or alter ACPI generation logic. + +Example: + +```bash +git clone https://gitlab.com/qemu-project/qemu.git qemu-upstream +cd qemu-upstream +git checkout 3e0bcba1ca7d6607ca49a988d165f052a3a53323 +git apply /path/to/qemu-acpi/scripts/qemu-upstream-dump.patch +mkdir build && cd build +../configure --target-list=x86_64-softmmu --enable-kvm --enable-tcg \ + --disable-docs --disable-werror +ninja qemu-system-x86_64 +/path/to/qemu-acpi/scripts/differential-upstream.sh \ + ./qemu-system-x86_64 10.2.4 +``` + +There was no genuine upstream QEMU 11.1 release at the time of this audit, so +the `11.1` compatibility profile remains pinned to the production compatibility +fork rather than being described as an upstream release profile. + +## Versions past the newest profile + +A QEMU version newer than the newest profile here is generated with that +profile (`Compatibility::LATEST`) rather than rejected: most releases leave the +Q35 ACPI ABI untouched, and when one does change it the generated blobs stop +matching the measured ones, which is strictly more informative than refusing to +generate. Adding a profile therefore means adding its fixtures **and** moving +`Compatibility::LATEST` to it. diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-10.0-q35-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-10.0-q35-base.bin new file mode 100644 index 000000000..fd88344db Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-10.0-q35-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-10.0-q35-hotplug-off-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-10.0-q35-hotplug-off-base.bin new file mode 100644 index 000000000..4cf417295 Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-10.0-q35-hotplug-off-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-10.0-q35-hotplug-off-numa-pxb-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-10.0-q35-hotplug-off-numa-pxb-base.bin new file mode 100644 index 000000000..895cfbe71 Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-10.0-q35-hotplug-off-numa-pxb-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-10.0-q35-numa-pxb-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-10.0-q35-numa-pxb-base.bin new file mode 100644 index 000000000..716658a04 Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-10.0-q35-numa-pxb-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-11.0-q35-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-11.0-q35-base.bin new file mode 100644 index 000000000..096a05d67 Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-11.0-q35-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-11.0-q35-hotplug-off-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-11.0-q35-hotplug-off-base.bin new file mode 100644 index 000000000..222d064a0 Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-11.0-q35-hotplug-off-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-11.0-q35-hotplug-off-numa-pxb-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-11.0-q35-hotplug-off-numa-pxb-base.bin new file mode 100644 index 000000000..8a16b109c Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-11.0-q35-hotplug-off-numa-pxb-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-11.0-q35-numa-pxb-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-11.0-q35-numa-pxb-base.bin new file mode 100644 index 000000000..8e384dd9a Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-11.0-q35-numa-pxb-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-base.bin new file mode 100644 index 000000000..204497c2a Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-hotplug-off-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-hotplug-off-base.bin new file mode 100644 index 000000000..ed3945ec8 Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-hotplug-off-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-hotplug-off-numa-pxb-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-hotplug-off-numa-pxb-base.bin new file mode 100644 index 000000000..9f47f52ed Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-hotplug-off-numa-pxb-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-numa-one-nic-loader.bin b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-numa-one-nic-loader.bin new file mode 100644 index 000000000..21a26c268 Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-numa-one-nic-loader.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-numa-one-nic-rsdp.bin b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-numa-one-nic-rsdp.bin new file mode 100644 index 000000000..22504c3ec Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-numa-one-nic-rsdp.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-numa-pxb-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-numa-pxb-base.bin new file mode 100644 index 000000000..99708a49b Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-numa-pxb-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-one-nic-loader.bin b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-one-nic-loader.bin new file mode 100644 index 000000000..a3fc0ea95 Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-one-nic-loader.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-one-nic-rsdp.bin b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-one-nic-rsdp.bin new file mode 100644 index 000000000..3266dc229 Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-one-nic-rsdp.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-one-nic.bin b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-one-nic.bin new file mode 100644 index 000000000..2cea5f111 Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-one-nic.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-9.1-q35-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-9.1-q35-base.bin new file mode 100644 index 000000000..bac1c15a5 Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-9.1-q35-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-9.1-q35-hotplug-off-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-9.1-q35-hotplug-off-base.bin new file mode 100644 index 000000000..767d28c68 Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-9.1-q35-hotplug-off-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-9.1-q35-hotplug-off-numa-pxb-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-9.1-q35-hotplug-off-numa-pxb-base.bin new file mode 100644 index 000000000..75baae40b Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-9.1-q35-hotplug-off-numa-pxb-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-9.1-q35-numa-pxb-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-9.1-q35-numa-pxb-base.bin new file mode 100644 index 000000000..a7655531a Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-9.1-q35-numa-pxb-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-9.2-q35-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-9.2-q35-base.bin new file mode 100644 index 000000000..00e0b7170 Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-9.2-q35-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-9.2-q35-hotplug-off-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-9.2-q35-hotplug-off-base.bin new file mode 100644 index 000000000..732ec74cc Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-9.2-q35-hotplug-off-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-9.2-q35-hotplug-off-numa-pxb-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-9.2-q35-hotplug-off-numa-pxb-base.bin new file mode 100644 index 000000000..e52408aa2 Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-9.2-q35-hotplug-off-numa-pxb-base.bin differ diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-9.2-q35-numa-pxb-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-9.2-q35-numa-pxb-base.bin new file mode 100644 index 000000000..1c1f159d3 Binary files /dev/null and b/dstack/crates/qemu-acpi/fixtures/qemu-9.2-q35-numa-pxb-base.bin differ diff --git a/dstack/crates/qemu-acpi/reference/Dockerfile b/dstack/crates/qemu-acpi/reference/Dockerfile new file mode 100644 index 000000000..5a2b55f35 --- /dev/null +++ b/dstack/crates/qemu-acpi/reference/Dockerfile @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# SPDX-License-Identifier: Apache-2.0 + +FROM ubuntu:24.04 AS builder + +ARG QEMU_REPOSITORY=https://github.com/kvinwang/qemu-tdx.git +ARG QEMU_REVISION=9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195 + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates git ninja-build pkg-config python3 python3-venv \ + build-essential libglib2.0-dev libpixman-1-dev zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN git clone --filter=blob:none "${QEMU_REPOSITORY}" /src/qemu \ + && git -C /src/qemu checkout "${QEMU_REVISION}" + +WORKDIR /src/qemu/build-acpi +RUN CFLAGS='-O2 -DDUMP_ACPI_TABLES -Wno-builtin-macro-redefined -D__DATE__="" -D__TIME__="" -D__TIMESTAMP__=""' \ + LDFLAGS='-Wl,--build-id=none' \ + ../configure \ + --target-list=x86_64-softmmu \ + --disable-werror \ + --disable-docs \ + --disable-gtk \ + --disable-sdl \ + --disable-vnc \ + --disable-curses \ + --disable-tools \ + --disable-user \ + && ninja qemu-system-x86_64 + +FROM ubuntu:24.04 + +ARG QEMU_REVISION=9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195 +LABEL org.opencontainers.image.source="https://github.com/kvinwang/qemu-tdx" \ + org.opencontainers.image.revision="${QEMU_REVISION}" \ + org.opencontainers.image.licenses="GPL-2.0-or-later" + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libglib2.0-0 libpixman-1-0 zlib1g \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /src/qemu/build-acpi/qemu-system-x86_64 /usr/local/bin/dstack-acpi-tables +COPY --from=builder /src/qemu/pc-bios /usr/share/qemu + +ENTRYPOINT ["/usr/local/bin/dstack-acpi-tables", "-L", "/usr/share/qemu", "-bios", "/usr/share/qemu/bios-256k.bin"] diff --git a/dstack/crates/qemu-acpi/reference/README.md b/dstack/crates/qemu-acpi/reference/README.md new file mode 100644 index 000000000..5eabc004e --- /dev/null +++ b/dstack/crates/qemu-acpi/reference/README.md @@ -0,0 +1,18 @@ +# dstack ACPI reference image + +This directory builds the production QEMU compatibility fork at revision +`9de6fdfff3a84103b83ca6b2e8c4fb8e05cf9195` with its test-only +`DUMP_ACPI_TABLES` mode. The resulting command writes QEMU's 128 KiB +`etc/acpi/tables` blob to standard output and exits without starting a VM. + +The image is a differential-test oracle only. Production Rust code does not +depend on it. The source revision and GPL license are recorded as OCI labels; +the corresponding source is available from the repository and revision named +in the labels. + +Build locally with: + +```sh +docker build -t kvin/dstack-acpi-tables:qemu-11.1 \ + -f dstack/crates/qemu-acpi/reference/Dockerfile . +``` diff --git a/dstack/crates/qemu-acpi/scripts/differential-qemu.sh b/dstack/crates/qemu-acpi/scripts/differential-qemu.sh new file mode 100755 index 000000000..860dac4b1 --- /dev/null +++ b/dstack/crates/qemu-acpi/scripts/differential-qemu.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +if (($# != 2)); then + echo "usage: $0 QEMU_SYSTEM_X86_64 DSTACK_IMAGE_DIR" >&2 + exit 2 +fi +qemu=$(realpath "$1") +image_dir=$(realpath "$2") +if [[ ! -x "$qemu" || ! -f "$image_dir/ovmf.fd" || ! -f "$image_dir/bzImage" ]]; then + echo "QEMU binary or dstack image inputs are invalid" >&2 + exit 2 +fi + +crate_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +workspace=$(cd -- "$crate_dir/../.." && pwd) +tmp=$(mktemp -d "${TMPDIR:-/tmp}/qemu-acpi-diff.XXXXXX") +trap 'rm -rf -- "$tmp"' EXIT + +(cd "$workspace" && cargo build -p qemu-acpi --example dump) +rust_dump="$workspace/target/debug/examples/dump" + +run_case() { + local version=$1 topology=$2 + local hugepages=0 gpus=0 + case "$topology" in + normal) ;; + numa) hugepages=1 ;; + numa-pxb) hugepages=1; gpus=8 ;; + *) echo "invalid internal topology: $topology" >&2; exit 2 ;; + esac + + local output="$tmp/$version-$topology" + mkdir "$output" + local args=( + -L "$(dirname "$qemu")/../pc-bios" + -cpu qemu64 -smp 8 -m 2048M -nographic -nodefaults -serial stdio + -bios "$image_dir/ovmf.fd" -kernel "$image_dir/bzImage" -initrd /bin/sh + -drive "file=/bin/sh,if=none,id=hd1,format=raw,readonly=on" + -device "virtio-blk-pci,drive=hd1" + -netdev "user,id=net0" -device "virtio-net-pci,netdev=net0" + -netdev "user,id=net1" -device "virtio-net-pci,netdev=net1" + -object "tdx-guest,id=tdx" -device "vhost-vsock-pci,guest-cid=3" + -virtfs "local,path=/bin,mount_tag=host-shared,readonly=on,security_model=none,id=virtfs0" + -drive "file=/bin/sh,if=none,id=hd0,format=raw,readonly=on" + -device "virtio-blk-pci,drive=hd0" + ) + local machine=q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off,smm=off,pic=off + if ((hugepages)); then + args+=( + -numa "node,nodeid=0,cpus=0-7,memdev=mem0" + -object "memory-backend-file,id=mem0,size=2048M,mem-path=/dev/hugepages,share=on,prealloc=no" + ) + fi + if ((gpus)); then + args+=(-object "iommufd,id=iommufd0" -device "pxb-pcie,id=pcie.node0,bus=pcie.0,addr=10,numa_node=0,bus_nr=5") + for ((index = 0; index < gpus; index++)); do + args+=( + -device "pcie-root-port,id=pci.$index,bus=pcie.node0,chassis=$index" + -device "vfio-pci,host=00:00.0,bus=pci.$index,iommufd=iommufd0" + ) + done + fi + + QEMU_ACPI_COMPAT_VER="$version" QEMU_ACPI_DUMP_DIR="$output" \ + "$qemu" "${args[@]}" -machine "$machine" + "$rust_dump" 2 8 "$version" "$gpus" 0 "$hugepages" + cmp "$output/tables.bin" /tmp/rust.bin + cmp "$output/loader.bin" /tmp/rust-loader.bin + cmp "$output/rsdp.bin" /tmp/rust-rsdp.bin + printf '%-6s %-8s tables+loader+rsdp match\n' "$version" "$topology" +} + +for version in 8.0.0 8.2.0 9.0.0 9.1.0 9.2.0 10.0.0 10.2.0 11.0.0 11.1.0 11.2.0; do + for topology in normal numa numa-pxb; do + run_case "$version" "$topology" + done +done diff --git a/dstack/crates/qemu-acpi/scripts/differential-random.py b/dstack/crates/qemu-acpi/scripts/differential-random.py new file mode 100755 index 000000000..409cb4c29 --- /dev/null +++ b/dstack/crates/qemu-acpi/scripts/differential-random.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# SPDX-License-Identifier: Apache-2.0 + +"""Reproducible differential tests against the QEMU ACPI reference image.""" + +# ruff: noqa: D101, D103 + +import argparse +import os +import random +import shutil +import struct +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path + +TABLE_BLOB_SIZE = 128 * 1024 +LOADER_COMMAND_SIZE = 128 +LOADER_BLOB_SIZE = 4096 +TABLES_FILE = "etc/acpi/tables" +RSDP_FILE = "etc/acpi/rsdp" + + +@dataclass(frozen=True) +class Case: + version: str = "11.1.0" + cpus: int = 1 + memory_mib: int = 2048 + nics: int = 1 + volumes: int = 0 + gpus: int = 0 + switches: int = 0 + hugepages: bool = False + root_verity: bool = True + hotplug_off: bool = False + smm: bool = False + pic: bool = False + pci_hole64_size: int = 0 + + +def fixed_cases(): + cases = [ + Case(version=v) + for v in ["8.0.0", "9.1.0", "9.2.0", "10.0.0", "11.0.0", "11.1.0", "11.2.0"] + ] + cases += [Case(cpus=n) for n in [1, 8, 9, 254, 255, 256, 4096]] + cases += [Case(memory_mib=n) for n in [1, 2047, 2048, 2815, 2816, 1_048_576]] + cases += [ + Case(nics=0, root_verity=False), + Case(nics=4, volumes=4), + Case(hotplug_off=True), + Case(smm=True, pic=True), + Case(pci_hole64_size=1 << 40), + Case(hugepages=True), + Case(hugepages=True, gpus=1), + Case(hugepages=True, gpus=8, switches=4), + Case(version="9.1.0", hugepages=True, gpus=1, hotplug_off=True), + ] + return cases + + +def random_case(rng): + hugepages = rng.choice([False, True]) + gpus = rng.randint(0, 8) + switches = rng.randint(0, 4) if gpus else 0 + pxb = hugepages and gpus > 0 + available = 25 if pxb else 26 + passthrough = switches if pxb else gpus + switches + root_verity = rng.choice([False, True]) + capacity = available - passthrough - (1 if root_verity else 0) + 1 + if pxb: + # QEMU adds the fixed-address PXB after the ordinary virtio devices. + # Keep slot 0x10 free for it, matching configurations QEMU can realize. + capacity = min(capacity, 12 - int(root_verity)) + nics = rng.randint(0, max(0, capacity)) + volumes = rng.randint(0, max(0, capacity - nics)) + return Case( + version=rng.choice( + ["8.0.0", "9.1.0", "9.2.0", "10.0.0", "11.0.0", "11.1.0", "11.2.0"] + ), + cpus=rng.choice([1, 2, 8, 9, 254, 255, 256, rng.randint(1, 512)]), + memory_mib=rng.choice([1, 2047, 2048, 2815, 2816, rng.randint(1, 1_048_576)]), + nics=nics, + volumes=volumes, + gpus=gpus, + switches=switches, + hugepages=hugepages, + root_verity=root_verity, + hotplug_off=rng.choice([False, True]), + smm=rng.choice([False, True]), + pic=rng.choice([False, True]), + pci_hole64_size=rng.choice([0, 32 << 30, 1 << 40]), + ) + + +def qemu_args(case): + args = [ + "-cpu", + "qemu64", + "-smp", + str(case.cpus), + "-m", + f"{case.memory_mib}M", + "-nographic", + "-nodefaults", + "-serial", + "stdio", + "-drive", + "file=/bin/sh,if=none,id=hd1,format=raw,readonly=on", + "-device", + "virtio-blk-pci,drive=hd1", + ] + for index in range(case.volumes): + args += [ + "-drive", + f"file=/bin/sh,if=none,id=vol{index},format=raw,readonly=on", + "-device", + f"virtio-blk-pci,drive=vol{index}", + ] + for index in range(case.nics): + args += [ + "-netdev", + f"socket,id=net{index},listen=:0", + "-device", + f"virtio-net-pci,netdev=net{index}", + ] + args += [ + "-device", + "vhost-vsock-pci,guest-cid=3", + "-virtfs", + "local,path=/bin,mount_tag=host-shared,readonly=on,security_model=none,id=virtfs0", + ] + if case.root_verity: + args += [ + "-drive", + "file=/bin/sh,if=none,id=hd0,format=raw,readonly=on", + "-device", + "virtio-blk-pci,drive=hd0", + ] + else: + args += ["-cdrom", "/bin/sh"] + args += [ + "-machine", + f"q35,kernel-irqchip=split,hpet=off,smm={'on' if case.smm else 'off'},pic={'on' if case.pic else 'off'}", + ] + if case.hugepages: + args += [ + "-numa", + f"node,nodeid=0,cpus=0-{case.cpus - 1},memdev=mem0", + "-object", + f"memory-backend-file,id=mem0,size={case.memory_mib}M,mem-path=/tmp,share=on,prealloc=no", + ] + port = 0 + if case.gpus: + args += ["-object", "iommufd,id=iommufd0"] + bus = "pcie.0" + if case.hugepages: + args += [ + "-device", + "pxb-pcie,id=pcie.node0,bus=pcie.0,addr=10,numa_node=0,bus_nr=5", + ] + bus = "pcie.node0" + for _ in range(case.gpus): + args += [ + "-device", + f"pcie-root-port,id=pci.{port},bus={bus},chassis={port}", + "-device", + f"vfio-pci,host=00:00.0,bus=pci.{port},iommufd=iommufd0", + ] + port += 1 + for _ in range(case.switches): + args += [ + "-device", + f"pcie-root-port,id=pci.{port},bus=pcie.0,chassis={port}", + "-device", + f"vfio-pci,host=00:00.0,bus=pci.{port},iommufd=iommufd0", + ] + port += 1 + if case.hotplug_off: + args += ["-global", "ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off"] + if case.pci_hole64_size: + args += ["-global", f"q35-pcihost.pci-hole64-size=0x{case.pci_hole64_size:x}"] + return args + + +def command(kind, payload): + data = struct.pack("<I", kind) + payload + return data + bytes(LOADER_COMMAND_SIZE - len(data)) + + +def filename(name): + encoded = name.encode() + return encoded + bytes(56 - len(encoded)) + + +def reference_blobs(tables): + if len(tables) < TABLE_BLOB_SIZE or len(tables) % TABLE_BLOB_SIZE: + raise RuntimeError( + f"reference emitted invalid table allocation size {len(tables)}" + ) + locations = {} + offset = 0 + while tables[offset : offset + 4] != bytes(4): + signature = tables[offset : offset + 4] + length = struct.unpack_from("<I", tables, offset + 4)[0] + if length < 36 or offset + length > len(tables): + raise RuntimeError( + f"invalid reference table {signature!r} length {length} at {offset}" + ) + locations[signature] = (offset, length) + offset += length + rsdt_offset, rsdt_length = locations[b"RSDT"] + rsdp = b"RSD PTR \0BOCHS \0" + struct.pack("<I", rsdt_offset) + loader = command(1, filename(RSDP_FILE) + struct.pack("<IB", 16, 2)) + loader += command(1, filename(TABLES_FILE) + struct.pack("<IB", 64, 1)) + + def checksum(signature): + start, length = locations[signature] + return command( + 3, filename(TABLES_FILE) + struct.pack("<III", start + 9, start, length) + ) + + loader += checksum(b"DSDT") + fadt, _ = locations[b"FACP"] + for relative, size in [(36, 4), (40, 4), (140, 8)]: + loader += command( + 2, + filename(TABLES_FILE) + + filename(TABLES_FILE) + + struct.pack("<IB", fadt + relative, size), + ) + loader += checksum(b"FACP") + checksum(b"APIC") + if b"SRAT" in locations: + loader += checksum(b"SRAT") + loader += checksum(b"MCFG") + checksum(b"WAET") + for relative in range(36, rsdt_length, 4): + loader += command( + 2, + filename(TABLES_FILE) + + filename(TABLES_FILE) + + struct.pack("<IB", rsdt_offset + relative, 4), + ) + loader += checksum(b"RSDT") + loader += command( + 2, filename(RSDP_FILE) + filename(TABLES_FILE) + struct.pack("<IB", 16, 4) + ) + loader += command(3, filename(RSDP_FILE) + struct.pack("<III", 8, 0, 20)) + return tables, loader + bytes(LOADER_BLOB_SIZE - len(loader)), rsdp + + +def first_difference(left, right): + for index, pair in enumerate(zip(left, right)): + if pair[0] != pair[1]: + return index + return min(len(left), len(right)) + + +def run_case(case, image, rust_dump, output): + env = os.environ.copy() + env["QEMU_ACPI_COMPAT_VER"] = case.version + reference = subprocess.run( + ["docker", "run", "--rm", "-e", f"QEMU_ACPI_COMPAT_VER={case.version}", image] + + qemu_args(case), + check=True, + stdout=subprocess.PIPE, + ).stdout + (output / "reference-tables.bin").write_bytes(reference) + subprocess.run( + [ + rust_dump, + str(case.nics), + str(case.cpus), + case.version, + str(case.gpus), + str(case.switches), + str(int(case.hugepages)), + str(int(case.root_verity)), + str(int(case.hotplug_off)), + str(int(case.smm)), + str(case.pci_hole64_size), + str(case.memory_mib * 1024 * 1024), + str(case.volumes), + str(int(case.pic)), + ], + check=True, + env={**env, "QEMU_ACPI_OUTPUT_DIR": str(output)}, + ) + expected = reference_blobs(reference) + actual = tuple( + (output / name).read_bytes() + for name in ["tables.bin", "loader.bin", "rsdp.bin"] + ) + for name, blob in zip(["tables", "loader", "rsdp"], expected): + (output / f"reference-{name}.bin").write_bytes(blob) + for name, wanted, got in zip(["tables", "loader", "rsdp"], expected, actual): + if wanted != got: + at = first_difference(wanted, got) + raise RuntimeError( + f"{name} differs at byte {at}: expected {wanted[at : at + 16].hex(' ')}, got {got[at : at + 16].hex(' ')}" + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--image", required=True) + parser.add_argument("--rust-dump", required=True) + parser.add_argument("--seed", type=lambda value: int(value, 0), default=0x1053) + parser.add_argument("--cases", type=int, default=32) + parser.add_argument("--failure-dir", type=Path) + args = parser.parse_args() + rng = random.Random(args.seed) + cases = fixed_cases() + [random_case(rng) for _ in range(args.cases)] + with tempfile.TemporaryDirectory(prefix="qemu-acpi-diff-") as directory: + output = Path(directory) + for index, case in enumerate(cases): + try: + run_case(case, args.image, args.rust_dump, output) + except Exception as error: + print( + f"FAIL seed={args.seed:#x} case={index} config={case}", + file=sys.stderr, + ) + if args.failure_dir: + if args.failure_dir.exists(): + shutil.rmtree(args.failure_dir) + shutil.copytree(output, args.failure_dir) + (args.failure_dir / "case.txt").write_text( + f"seed={args.seed:#x}\nindex={index}\nconfig={case}\nerror={error}\n" + ) + raise + print(f"PASS {index + 1}/{len(cases)} {case}") + + +if __name__ == "__main__": + main() diff --git a/dstack/crates/qemu-acpi/scripts/differential-upstream.sh b/dstack/crates/qemu-acpi/scripts/differential-upstream.sh new file mode 100755 index 000000000..1a6773f2a --- /dev/null +++ b/dstack/crates/qemu-acpi/scripts/differential-upstream.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +if (($# != 2)); then + echo "usage: $0 QEMU_SYSTEM_X86_64 QEMU_VERSION" >&2 + exit 2 +fi +qemu=$(realpath "$1") +version=$2 +if [[ ! -x "$qemu" || ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "invalid QEMU binary or semantic version" >&2 + exit 2 +fi +crate_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +workspace=$(cd -- "$crate_dir/../.." && pwd) +tmp=$(mktemp -d "${TMPDIR:-/tmp}/qemu-acpi-upstream.XXXXXX") +trap 'rm -rf -- "$tmp"' EXIT +(cd "$workspace" && cargo build -p qemu-acpi --example dump) +rust_dump="$workspace/target/debug/examples/dump" + +for topology in normal numa numa-pxb; do + hugepages=0 + gpus=0 + case "$topology" in + normal) ;; + numa) hugepages=1 ;; + numa-pxb) hugepages=1; gpus=8 ;; + esac + output="$tmp/$topology" + mkdir "$output" + args=( + -cpu qemu64 -smp 8 -m 2048M -nographic -nodefaults -serial stdio + -drive "file=/bin/sh,if=none,id=hd1,format=raw,readonly=on" + -device "virtio-blk-pci,drive=hd1" + -netdev "socket,id=net0,listen=:0" -device "virtio-net-pci,netdev=net0" + -netdev "socket,id=net1,listen=:0" -device "virtio-net-pci,netdev=net1" + -object "tdx-guest,id=tdx" -device "vhost-vsock-pci,guest-cid=3" + -virtfs "local,path=/bin,mount_tag=host-shared,readonly=on,security_model=none,id=virtfs0" + -drive "file=/bin/sh,if=none,id=hd0,format=raw,readonly=on" + -device "virtio-blk-pci,drive=hd0" + ) + machine=q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off,smm=off,pic=off + if ((hugepages)); then + args+=( + -numa "node,nodeid=0,cpus=0-7,memdev=mem0" + -object "memory-backend-ram,id=mem0,size=2048M" + ) + fi + if ((gpus)); then + args+=(-device "pxb-pcie,id=pcie.node0,bus=pcie.0,addr=10,numa_node=0,bus_nr=5") + for ((index = 0; index < gpus; index++)); do + # The endpoint does not emit AML. pci-testdev avoids requiring real + # VFIO hardware while preserving QEMU's root-port topology. + args+=( + -device "pcie-root-port,id=pci.$index,bus=pcie.node0,chassis=$index" + -device "pci-testdev,bus=pci.$index" + ) + done + fi + QEMU_DUMP_ACPI_DIR="$output" "$qemu" "${args[@]}" -machine "$machine" + "$rust_dump" 2 8 "$version" "$gpus" 0 "$hugepages" + cmp "$output/tables.bin" /tmp/rust.bin + cmp "$output/loader.bin" /tmp/rust-loader.bin + cmp "$output/rsdp.bin" /tmp/rust-rsdp.bin + printf '%-8s %-8s tables+loader+rsdp match\n' "$version" "$topology" +done diff --git a/dstack/crates/qemu-acpi/scripts/qemu-dump-all-blobs.patch b/dstack/crates/qemu-acpi/scripts/qemu-dump-all-blobs.patch new file mode 100644 index 000000000..a14e58a9c --- /dev/null +++ b/dstack/crates/qemu-acpi/scripts/qemu-dump-all-blobs.patch @@ -0,0 +1,43 @@ +From: dstack ACPI differential harness +Subject: [TEST ONLY] dump all measured ACPI fw_cfg blobs +SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +SPDX-License-Identifier: Apache-2.0 + +This instrumentation does not alter ACPI construction. When +QEMU_ACPI_DUMP_DIR is set, it writes the three measured blobs and exits. +Without that variable, the existing DUMP_ACPI_TABLES behavior is unchanged. + +--- a/hw/i386/acpi-build.c ++++ b/hw/i386/acpi-build.c +@@ -2775,6 +2775,31 @@ void acpi_setup(void) + + #ifdef DUMP_ACPI_TABLES + { ++ const char *dump_dir = getenv("QEMU_ACPI_DUMP_DIR"); ++ if (dump_dir) { ++ struct { ++ const char *name; ++ GArray *blob; ++ } outputs[] = { ++ { "tables.bin", tables.table_data }, ++ { "loader.bin", tables.linker->cmd_blob }, ++ { "rsdp.bin", tables.rsdp }, ++ }; ++ ++ for (size_t i = 0; i < ARRAY_SIZE(outputs); i++) { ++ g_autofree char *path = g_build_filename(dump_dir, ++ outputs[i].name, ++ NULL); ++ g_autoptr(GError) error = NULL; ++ if (!g_file_set_contents(path, outputs[i].blob->data, ++ outputs[i].blob->len, &error)) { ++ error_report("failed to dump %s: %s", path, ++ error->message); ++ exit(1); ++ } ++ } ++ exit(0); ++ } + uint8_t *ptr = (uint8_t *)tables.table_data->data; + uint64_t size = tables.table_data->len; + int flags = fcntl(1, F_GETFL); diff --git a/dstack/crates/qemu-acpi/scripts/qemu-upstream-dump.patch b/dstack/crates/qemu-acpi/scripts/qemu-upstream-dump.patch new file mode 100644 index 000000000..fd02817ef --- /dev/null +++ b/dstack/crates/qemu-acpi/scripts/qemu-upstream-dump.patch @@ -0,0 +1,130 @@ +SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +SPDX-License-Identifier: Apache-2.0 + +Test-only upstream QEMU instrumentation used to compare the production helper +ABI. QEMU_DUMP_ACPI_DIR normalizes the same dynamic platform values as the +production DUMP_ACPI_TABLES patch, bypasses TDX/KVM realization requirements, +dumps all three measured fw_cfg blobs, and exits. It does not change the ACPI +builders or version-compatibility logic being tested. + +diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c +index 9446a9f862..be1a01dbc2 100644 +--- a/hw/i386/acpi-build.c ++++ b/hw/i386/acpi-build.c +@@ -143,7 +143,8 @@ static void init_common_fadt_data(MachineState *ms, Object *o, + */ + bool smm_enabled = object_property_get_bool(o, "smm-compat", NULL) ? + true : x86_machine_is_smm_enabled(x86ms); +- uint32_t io = object_property_get_uint(o, ACPI_PM_PROP_PM_IO_BASE, NULL); ++ uint32_t io = getenv("QEMU_DUMP_ACPI_DIR") ? 0x600 : ++ object_property_get_uint(o, ACPI_PM_PROP_PM_IO_BASE, NULL); + AmlAddressSpace as = AML_AS_SYSTEM_IO; + AcpiFadtData fadt = { + .rev = 3, +@@ -164,7 +165,8 @@ static void init_common_fadt_data(MachineState *ms, Object *o, + .plvl2_lat = 0xfff /* C2 state not supported */, + .plvl3_lat = 0xfff /* C3 state not supported */, + .smi_cmd = smm_enabled ? ACPI_PORT_SMI_CMD : 0, +- .sci_int = object_property_get_uint(o, ACPI_PM_PROP_SCI_INT, NULL), ++ .sci_int = getenv("QEMU_DUMP_ACPI_DIR") ? 9 : ++ object_property_get_uint(o, ACPI_PM_PROP_SCI_INT, NULL), + .acpi_enable_cmd = + smm_enabled ? + object_property_get_uint(o, ACPI_PM_PROP_ACPI_ENABLE_CMD, NULL) : +@@ -179,7 +181,8 @@ static void init_common_fadt_data(MachineState *ms, Object *o, + .pm_tmr = { .space_id = as, .bit_width = 4 * 8, .address = io + 0x08 }, + .gpe0_blk = { .space_id = as, .bit_width = + object_property_get_uint(o, ACPI_PM_PROP_GPE0_BLK_LEN, NULL) * 8, +- .address = object_property_get_uint(o, ACPI_PM_PROP_GPE0_BLK, NULL) ++ .address = getenv("QEMU_DUMP_ACPI_DIR") ? io + 0x20 : ++ object_property_get_uint(o, ACPI_PM_PROP_GPE0_BLK, NULL) + }, + }; + +@@ -295,6 +298,16 @@ static void acpi_get_pci_holes(Range *hole, Range *hole64) + return; + } + ++ if (getenv("QEMU_DUMP_ACPI_DIR")) { ++ X86MachineState *x86ms = X86_MACHINE(qdev_get_machine()); ++ uint64_t hole64_size = object_property_get_uint( ++ pci_host, PCI_HOST_PROP_PCI_HOLE64_SIZE, NULL); ++ range_set_bounds1(hole, x86ms->below_4g_mem_size, 0xfec00000); ++ range_set_bounds1(hole64, 0x380000000000ULL, ++ 0x380000000000ULL + hole64_size); ++ return; ++ } ++ + range_set_bounds1(hole, + object_property_get_uint(pci_host, + PCI_HOST_PROP_PCI_HOLE_START, +@@ -1895,6 +1908,12 @@ struct AcpiBuildState { + static bool acpi_get_mcfg(AcpiMcfgInfo *mcfg) + { + Object *pci_host; ++ ++ if (getenv("QEMU_DUMP_ACPI_DIR")) { ++ mcfg->base = 0xe0000000; ++ mcfg->size = 0x10000000; ++ return true; ++ } + QObject *o; + + pci_host = acpi_get_i386_pci_host(); +@@ -2204,6 +2223,34 @@ void acpi_setup(void) + acpi_build_tables_init(&tables); + acpi_build(&tables, MACHINE(pcms)); + ++ { ++ const char *dump_dir = getenv("QEMU_DUMP_ACPI_DIR"); ++ struct { ++ const char *name; ++ GArray *blob; ++ } outputs[] = { ++ { "tables.bin", tables.table_data }, ++ { "loader.bin", tables.linker->cmd_blob }, ++ { "rsdp.bin", tables.rsdp }, ++ }; ++ ++ if (dump_dir) { ++ for (size_t i = 0; i < ARRAY_SIZE(outputs); i++) { ++ g_autofree char *path = g_build_filename( ++ dump_dir, outputs[i].name, NULL); ++ g_autoptr(GError) error = NULL; ++ ++ if (!g_file_set_contents(path, outputs[i].blob->data, ++ outputs[i].blob->len, &error)) { ++ error_report("failed to dump %s: %s", path, ++ error->message); ++ exit(1); ++ } ++ } ++ exit(0); ++ } ++ } ++ + /* Now expose it all to Guest */ + build_state->table_mr = acpi_add_rom_blob(acpi_build_update, + build_state, tables.table_data, +diff --git a/target/i386/kvm/tdx.c b/target/i386/kvm/tdx.c +index 7dcf5b4e4e..f2f17f7dcb 100644 +--- a/target/i386/kvm/tdx.c ++++ b/target/i386/kvm/tdx.c +@@ -748,7 +748,7 @@ static void tdx_cpu_instance_init(X86ConfidentialGuest *cg, CPUState *cpu) + X86CPUClass *xcc = X86_CPU_GET_CLASS(cpu); + X86CPU *x86cpu = X86_CPU(cpu); + +- if (xcc->model) { ++ if (xcc->model && !getenv("QEMU_DUMP_ACPI_DIR")) { + error_report("Named cpu model is not supported for TDX yet!"); + exit(1); + } +@@ -1505,7 +1505,7 @@ static void tdx_guest_init(Object *obj) + + qemu_mutex_init(&tdx->lock); + +- cgs->require_guest_memfd = true; ++ cgs->require_guest_memfd = getenv("QEMU_DUMP_ACPI_DIR") == NULL; + tdx->attributes = TDX_TD_ATTRIBUTES_SEPT_VE_DISABLE; + + object_property_add_uint64_ptr(obj, "attributes", &tdx->attributes, diff --git a/dstack/crates/qemu-acpi/src/aml_encode.rs b/dstack/crates/qemu-acpi/src/aml_encode.rs new file mode 100644 index 000000000..0a34efdf9 --- /dev/null +++ b/dstack/crates/qemu-acpi/src/aml_encode.rs @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// SPDX-License-Identifier: Apache-2.0 +use crate::Error; +use alloc::vec::Vec; +extern crate alloc; +fn encode_pkg_length(value: usize) -> Result<Vec<u8>, Error> { + if value < 0x40 { + return Ok(vec![value as u8]); + } + let follow = if value < 0x1000 { + 1 + } else if value < 0x100000 { + 2 + } else if value < 0x10000000 { + 3 + } else { + return Err(Error::MalformedTables("AML package exceeds 256 MiB".into())); + }; + let mut out = vec![((follow << 6) as u8) | (value as u8 & 0x0f)]; + for index in 0..follow { + out.push((value >> (4 + index * 8)) as u8); + } + Ok(out) +} +pub(crate) fn integer(value: u32) -> Vec<u8> { + match value { + 0 => vec![0], + 1 => vec![1], + 2..=0xff => vec![0x0a, value as u8], + 0x100..=0xffff => { + let mut out = vec![0x0b]; + out.extend_from_slice(&(value as u16).to_le_bytes()); + out + } + _ => { + let mut out = vec![0x0c]; + out.extend_from_slice(&value.to_le_bytes()); + out + } + } +} +pub(crate) fn package(opcode: &[u8], body: &[u8]) -> Result<Vec<u8>, Error> { + let mut total = body + .len() + .checked_add(1) + .ok_or_else(|| Error::MalformedTables("AML package size overflow".into()))?; + loop { + let length = encode_pkg_length(total)?; + let adjusted = body + .len() + .checked_add(length.len()) + .ok_or_else(|| Error::MalformedTables("AML package size overflow".into()))?; + if adjusted == total { + let capacity = opcode + .len() + .checked_add(total) + .ok_or_else(|| Error::MalformedTables("AML package size overflow".into()))?; + let mut out = Vec::with_capacity(capacity); + out.extend_from_slice(opcode); + out.extend_from_slice(&length); + out.extend_from_slice(body); + return Ok(out); + } + total = adjusted; + } +} diff --git a/dstack/crates/qemu-acpi/src/cpu.rs b/dstack/crates/qemu-acpi/src/cpu.rs new file mode 100644 index 000000000..3adad2ecb --- /dev/null +++ b/dstack/crates/qemu-acpi/src/cpu.rs @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// SPDX-License-Identifier: Apache-2.0 + +use crate::aml_encode::{integer, package}; +use crate::Error; + +fn hex_digit(value: u32) -> u8 { + match value & 0xf { + 0..=9 => b'0' + (value & 0xf) as u8, + _ => b'A' + ((value & 0xf) as u8 - 10), + } +} +fn name(index: u32) -> [u8; 4] { + [ + b'C', + hex_digit(index >> 8), + hex_digit(index >> 4), + hex_digit(index), + ] +} + +fn method(method_name: &[u8; 4], flags: u8, terms: &[u8]) -> Result<Vec<u8>, Error> { + let mut body = Vec::with_capacity(5 + terms.len()); + body.extend_from_slice(method_name); + body.push(flags); + body.extend_from_slice(terms); + package(&[0x14], &body) +} + +fn status(index: u32) -> Result<Vec<u8>, Error> { + let mut terms = vec![0xa4]; // Return + terms.extend_from_slice(b"CSTA"); + terms.extend_from_slice(&integer(index)); + method(b"_STA", 0x08, &terms) +} + +fn mat(index: u32, x2apic: bool) -> Result<Vec<u8>, Error> { + let mut entry = Vec::new(); + if x2apic { + entry.extend_from_slice(&[9, 16, 0, 0]); + entry.extend_from_slice(&index.to_le_bytes()); + entry.extend_from_slice(&1u32.to_le_bytes()); + entry.extend_from_slice(&index.to_le_bytes()); + } else { + entry.extend_from_slice(&[0, 8, index as u8, index as u8]); + entry.extend_from_slice(&1u32.to_le_bytes()); + } + let mut buffer = integer(entry.len() as u32); + buffer.extend_from_slice(&entry); + let buffer = package(&[0x11], &buffer)?; + let mut out = vec![0x08]; + out.extend_from_slice(b"_MAT"); + out.extend_from_slice(&buffer); + Ok(out) +} + +fn eject(index: u32) -> Result<Vec<u8>, Error> { + let mut call = b"CEJ0".to_vec(); + call.extend_from_slice(&integer(index)); + method(b"_EJ0", 1, &call) +} + +fn ost(index: u32) -> Result<Vec<u8>, Error> { + let mut call = b"COST".to_vec(); + call.extend_from_slice(&integer(index)); + call.extend_from_slice(&[0x68, 0x69, 0x6a]); + method(b"_OST", 0x0b, &call) +} + +pub(crate) fn object(index: u32, numa: bool) -> Result<Vec<u8>, Error> { + let x2apic = index >= 255; + let mut body = Vec::new(); + body.extend_from_slice(&name(index)); + if x2apic { + body.extend_from_slice(&[0x08]); + body.extend_from_slice(b"_HID"); + body.extend_from_slice(&[0x0d]); + body.extend_from_slice(b"ACPI0007\0"); + body.extend_from_slice(&[0x08]); + body.extend_from_slice(b"_UID"); + body.extend_from_slice(&integer(index)); + } else { + body.push(index as u8); // ProcID + body.extend_from_slice(&0u32.to_le_bytes()); // PblkAddr + body.push(0); // PblkLen + } + body.extend_from_slice(&status(index)?); + body.extend_from_slice(&mat(index, x2apic)?); + if index != 0 { + body.extend_from_slice(&eject(index)?); + } + body.extend_from_slice(&ost(index)?); + if numa { + body.extend_from_slice(&[0x08, b'_', b'P', b'X', b'M', 0x00]); + } + package(if x2apic { &[0x5b, 0x82] } else { &[0x5b, 0x83] }, &body) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn legacy_processor_matches_qemu() -> Result<(), Error> { + let Ok(expected) = hex::decode("5b83450443303031010000000000140c5f53544108a44353544101085f4d4154110b0a080008010101000000140b5f454a300143454a3001140e5f4f53540b434f53540168696a") else { panic!("valid test vector") }; + assert_eq!(object(1, false)?, expected); + Ok(()) + } + + #[test] + fn multi_byte_integer_processor_matches_qemu() -> Result<(), Error> { + let Ok(expected) = hex::decode("5b83480443303032020000000000140d5f53544108a4435354410a02085f4d4154110b0a080008020201000000140c5f454a300143454a300a02140f5f4f53540b434f53540a0268696a") else { panic!("valid test vector") }; + assert_eq!(object(2, false)?, expected); + Ok(()) + } +} diff --git a/dstack/crates/qemu-acpi/src/dsdt/cpus.rs b/dstack/crates/qemu-acpi/src/dsdt/cpus.rs new file mode 100644 index 000000000..730326e4b --- /dev/null +++ b/dstack/crates/qemu-acpi/src/dsdt/cpus.rs @@ -0,0 +1,609 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! CPU hotplug scaffolding: the `PRES` register window and the `CPUS` +//! container with one processor object per possible CPU. +//! +//! This is QEMU's `build_cpus_aml()` (`hw/acpi/cpu.c`). The two devices are +//! emitted bare here; the caller wraps them in `Scope (_SB)`. +//! +//! ```asl +//! Device (\_SB.PCI0.PRES) { +//! Name (_HID, EisaId ("PNP0A06")) +//! Name (_UID, "CPU Hotplug resources") +//! Mutex (CPLK, 0x00) +//! Name (_CRS, ResourceTemplate () { +//! IO (Decode16, 0x0CD8, 0x0CD8, 0x01, 0x0C) +//! }) +//! OperationRegion (PRST, SystemIO, 0x0CD8, 0x0C) +//! Field (PRST, ByteAcc, NoLock, WriteAsZeros) { +//! Offset (0x04), CPEN, 1, CINS, 1, CRMV, 1, CEJ0, 1, CEJF, 1, +//! Offset (0x05), CCMD, 8 +//! } +//! Field (PRST, DWordAcc, NoLock, Preserve) { +//! CSEL, 32, Offset (0x08), CDAT, 32 +//! } +//! } +//! Device (\_SB.CPUS) { +//! Name (_HID, "ACPI0010") +//! Name (_CID, EisaId ("PNP0A05")) +//! Method (CTFY, 2, NotSerialized) { +//! If ((Arg0 == Zero)) { Notify (C000, Arg1) } +//! } +//! Method (CSTA, 1, Serialized) { +//! Acquire (\_SB.PCI0.PRES.CPLK, 0xFFFF) +//! \_SB.PCI0.PRES.CSEL = Arg0 +//! Local0 = Zero +//! If ((\_SB.PCI0.PRES.CPEN == One)) { Local0 = 0x0F } +//! Release (\_SB.PCI0.PRES.CPLK) +//! Return (Local0) +//! } +//! Method (CEJ0, 1, Serialized) { +//! Acquire (\_SB.PCI0.PRES.CPLK, 0xFFFF) +//! \_SB.PCI0.PRES.CSEL = Arg0 +//! \_SB.PCI0.PRES.CEJ0 = One +//! Release (\_SB.PCI0.PRES.CPLK) +//! } +//! Method (CSCN, 0, Serialized) { +//! Acquire (\_SB.PCI0.PRES.CPLK, 0xFFFF) +//! Name (CNEW, Package (0xFF) {}) +//! Name (CEJL, Package (0xFF) {}) +//! Local3 = Zero +//! Local4 = One +//! While ((Local4 == One)) { +//! Local4 = Zero +//! Local0 = One +//! Local1 = Zero +//! Local5 = Zero +//! While (((Local0 == One) && (Local3 < One))) { +//! Local0 = Zero +//! \_SB.PCI0.PRES.CSEL = Local3 +//! \_SB.PCI0.PRES.CCMD = Zero +//! If ((\_SB.PCI0.PRES.CDAT < Local3)) { Break } +//! If (((Local1 == 0xFF) || (Local5 == 0xFF))) { +//! Local4 = One +//! Break +//! } +//! Local3 = \_SB.PCI0.PRES.CDAT +//! If ((\_SB.PCI0.PRES.CINS == One)) { +//! CNEW [Local1] = Local3 +//! Local1++ +//! Local0 = One +//! } +//! If ((\_SB.PCI0.PRES.CRMV == One)) { +//! CEJL [Local5] = Local3 +//! Local5++ +//! Local0 = One +//! } +//! Local3++ +//! } +//! Local2 = Zero +//! While ((Local2 < Local1)) { +//! Local3 = DerefOf (CNEW [Local2]) +//! CTFY (Local3, One) +//! Debug = Local3 +//! \_SB.PCI0.PRES.CSEL = Local3 +//! \_SB.PCI0.PRES.CINS = One +//! Local2++ +//! } +//! Local2 = Zero +//! While ((Local2 < Local5)) { +//! Local3 = DerefOf (CEJL [Local2]) +//! CTFY (Local3, 0x03) +//! \_SB.PCI0.PRES.CSEL = Local3 +//! \_SB.PCI0.PRES.CRMV = One +//! Local2++ +//! } +//! } +//! Release (\_SB.PCI0.PRES.CPLK) +//! } +//! Method (COST, 4, Serialized) { +//! Acquire (\_SB.PCI0.PRES.CPLK, 0xFFFF) +//! \_SB.PCI0.PRES.CSEL = Arg0 +//! \_SB.PCI0.PRES.CCMD = One +//! \_SB.PCI0.PRES.CDAT = Arg1 +//! \_SB.PCI0.PRES.CCMD = 0x02 +//! \_SB.PCI0.PRES.CDAT = Arg2 +//! Release (\_SB.PCI0.PRES.CPLK) +//! } +//! Processor (C000, 0x00, 0x00000000, 0x00) { +//! Method (_STA, 0, Serialized) { Return (CSTA (Zero)) } +//! Name (_MAT, Buffer (0x08) { 0x00, 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 }) +//! Method (_OST, 3, Serialized) { COST (Zero, Arg0, Arg1, Arg2) } +//! } +//! } +//! ``` + +use acpi_tables::aml::{ + Acquire, Arg, DeRefOf, Device, EISAName, Else, Equal, Field, FieldAccessType, FieldEntry, + FieldLockRule, FieldUpdateRule, If, Index, LessThan, Local, Method, MethodCall, Mutex, Name, + Notify, One, OpRegion, OpRegionSpace, Path, Release, ResourceTemplate, Return, Store, While, + Zero, IO, +}; +use acpi_tables::{Aml, AmlSink}; + +use super::ops::{emit, pkg_length, Increment, Raw}; +use crate::Error; + +/// Base of the CPU hotplug register block (`ACPI_CPU_HOTPLUG_BASE`) and its +/// length (`ACPI_CPU_HOTPLUG_REG_LEN`). +const HOTPLUG_BASE: u16 = 0x0cd8; +const HOTPLUG_LEN: u8 = 12; + +/// Largest number of CPUs one scan pass can collect: the ACPI 1.0 `Package` +/// the scan method uses to cache them holds at most 255 elements. +const MAX_CPUS_PER_PASS: u8 = 255; + +/// `CPHP_GET_NEXT_CPU_WITH_EVENT_CMD`, `CPHP_OST_EVENT_CMD`, +/// `CPHP_OST_STATUS_CMD`: the values written to the command register. +const CMD_GET_NEXT_CPU: u8 = 0; +const CMD_OST_EVENT: u8 = 1; +const CMD_OST_STATUS: u8 = 2; + +/// `Notify()` reason codes: device check (a CPU appeared) and eject request. +const DEVICE_CHECK: u8 = 1; +const EJECT_REQUEST: u8 = 3; + +/// Device holding the hotplug registers. Every field below is addressed +/// through its full path, because the methods that use them live in a +/// sibling device. +const RES: &str = "\\_SB_.PCI0.PRES"; + +/// Path of one register field inside the hotplug resource device. +fn res(field: &str) -> Path { + Path::new(&format!("{RES}.{field}")) +} + +pub(crate) fn build( + cpu_count: u32, + numa: bool, + initialize_selector: bool, + queued_eject: bool, +) -> Result<Vec<u8>, Error> { + let mut out = resource_device(initialize_selector); + out.extend(cpus_device(cpu_count, numa, queued_eject)?); + Ok(out) +} + +/// `Device (\_SB.PCI0.PRES)`: the I/O window plus the fields overlaid on it. +fn resource_device(initialize_selector: bool) -> Vec<u8> { + let hid = Name::new(Path::new("_HID"), &EISAName::new("PNP0A06")); + let uid = Name::new(Path::new("_UID"), &"CPU Hotplug resources"); + let lock = Mutex::new(Path::new("CPLK"), 0); + + let io = IO::new(HOTPLUG_BASE, HOTPLUG_BASE, 1, HOTPLUG_LEN); + let crs = Name::new(Path::new("_CRS"), &ResourceTemplate::new(vec![&io])); + + let region = OpRegion::new( + Path::new("PRST"), + OpRegionSpace::SystemIO, + &HOTPLUG_BASE, + &HOTPLUG_LEN, + ); + + // Flag bits, byte-accessed: one bit each for enabled, insert event, + // remove event, eject request and firmware-eject request, then the + // command register. + let flags = Field::new( + Path::new("PRST"), + FieldAccessType::Byte, + FieldLockRule::NoLock, + FieldUpdateRule::WriteAsZeroes, + vec![ + FieldEntry::Reserved(4 * 8), + FieldEntry::Named(*b"CPEN", 1), + FieldEntry::Named(*b"CINS", 1), + FieldEntry::Named(*b"CRMV", 1), + FieldEntry::Named(*b"CEJ0", 1), + FieldEntry::Named(*b"CEJF", 1), + FieldEntry::Reserved(3), + FieldEntry::Named(*b"CCMD", 8), + ], + ); + + // The same window seen as two dwords: the CPU selector and the data + // register, with the flag/command bytes skipped. + let words = Field::new( + Path::new("PRST"), + FieldAccessType::DWord, + FieldLockRule::NoLock, + FieldUpdateRule::Preserve, + vec![ + FieldEntry::Named(*b"CSEL", 32), + FieldEntry::Reserved(4 * 8), + FieldEntry::Named(*b"CDAT", 32), + ], + ); + + let selector = Path::new("CSEL"); + let initialize = Store::new(&selector, &Zero {}); + let initialize = Method::new(Path::new("_INI"), 0, true, vec![&initialize]); + let initialize = initialize_selector.then(|| emit(&initialize)); + let initialize = Raw(match initialize.as_deref() { + Some(bytes) => bytes, + None => &[], + }); + + emit(&Device::new( + Path::new(RES), + vec![ + &hid as &dyn Aml, + &uid, + &lock, + &crs, + ®ion, + &flags, + &words, + &initialize, + ], + )) +} + +/// `Device (\_SB.CPUS)`: the control methods plus one object per CPU. +fn cpus_device(cpu_count: u32, numa: bool, queued_eject: bool) -> Result<Vec<u8>, Error> { + let hid = Name::new(Path::new("_HID"), &"ACPI0010"); + let cid = Name::new(Path::new("_CID"), &EISAName::new("PNP0A05")); + + let notify = notify_method(cpu_count); + let status = status_method(); + let eject = eject_method(); + let scan = scan_method(cpu_count, queued_eject); + let ost = ost_method(); + + let mut processors = Vec::new(); + for index in 0..cpu_count { + processors.extend(crate::cpu::object(index, numa)?); + } + + let methods = [notify, status, eject, scan, ost].concat(); + let (methods, processors) = (Raw(&methods), Raw(&processors)); + Ok(emit(&Device::new( + Path::new("\\_SB_.CPUS"), + vec![&hid as &dyn Aml, &cid, &methods, &processors], + ))) +} + +/// `Method (CTFY, 2)`: dispatch a notification to the named CPU object, +/// since `Notify` needs a literal name rather than a computed one. +fn notify_method(cpu_count: u32) -> Vec<u8> { + let mut cases = Vec::new(); + for index in 0..cpu_count { + let selected = Equal::new(&Arg(0), &index); + let cpu = Path::new(&cpu_name(index)); + let notify = Notify::new(&cpu, &Arg(1)); + cases.extend(emit(&If::new(&selected, vec![¬ify]))); + } + let cases = Raw(&cases); + emit(&Method::new(Path::new("CTFY"), 2, false, vec![&cases])) +} + +/// `Method (CSTA, 1)`: report whether the selected CPU is enabled. +fn status_method() -> Vec<u8> { + let status = Local(0); + let csel = res("CSEL"); + let cpen = res("CPEN"); + let acquire = Acquire::new(res("CPLK"), 0xffff); + let select = Store::new(&csel, &Arg(0)); + let clear = Store::new(&status, &Zero {}); + let enabled = Equal::new(&cpen, &One {}); + let present = Store::new(&status, &0x0fu8); + let check = If::new(&enabled, vec![&present]); + let release = Release::new(res("CPLK")); + let result = Return::new(&status); + emit(&Method::new( + Path::new("CSTA"), + 1, + true, + vec![&acquire, &select, &clear, &check, &release, &result], + )) +} + +/// `Method (CEJ0, 1)`: ask the hotplug controller to eject the selected CPU. +fn eject_method() -> Vec<u8> { + let csel = res("CSEL"); + let cej0 = res("CEJ0"); + let acquire = Acquire::new(res("CPLK"), 0xffff); + let select = Store::new(&csel, &Arg(0)); + let eject = Store::new(&cej0, &One {}); + let release = Release::new(res("CPLK")); + emit(&Method::new( + Path::new("CEJ0"), + 1, + true, + vec![&acquire, &select, &eject, &release], + )) +} + +/// `Method (CSCN)`: walk the CPUs with pending events, in batches of at most +/// `MAX_CPUS_PER_PASS`, notifying OSPM about each insert and each eject. +fn scan_method(cpu_count: u32, queued_eject: bool) -> Vec<u8> { + let (csel, ccmd, cdat) = (res("CSEL"), res("CCMD"), res("CDAT")); + let (cins, crmv) = (res("CINS"), res("CRMV")); + let (added_list, eject_list) = (Path::new("CNEW"), Path::new("CEJL")); + + let has_event = Local(0); + let num_added = Local(1); + let cpu_idx = Local(2); + let uid = Local(3); + let has_job = Local(4); + let num_eject = Local(5); + + let acquire = Acquire::new(res("CPLK"), 0xffff); + // Named packages, not locals: old Windows cannot hold a package in a + // local, and its packages are capped at 255 elements. + let declare_added = Name::new(Path::new("CNEW"), &FixedPackage(MAX_CPUS_PER_PASS)); + let declare_eject_term = Name::new(Path::new("CEJL"), &FixedPackage(MAX_CPUS_PER_PASS)); + let declare_eject_bytes = queued_eject.then(|| emit(&declare_eject_term)); + let declare_eject = Raw(match declare_eject_bytes.as_deref() { + Some(bytes) => bytes, + None => &[], + }); + let first_uid = Store::new(&uid, &Zero {}); + let arm_job = Store::new(&has_job, &One {}); + + // Inner loop: collect CPUs with events until the batch is full, the scan + // wraps around, or every CPU has been looked at. + let clear_event = Store::new(&has_event, &Zero {}); + let select = Store::new(&csel, &uid); + let next_cpu = Store::new(&ccmd, &CMD_GET_NEXT_CPU); + let wrapped = LessThan::new(&cdat, &uid); + let wrap_exit = If::new(&wrapped, vec![&Break]); + let added_full = Equal::new(&num_added, &MAX_CPUS_PER_PASS); + let eject_full = Equal::new(&num_eject, &MAX_CPUS_PER_PASS); + let both_full = LOr::new(&added_full, &eject_full); + let batch_full_bytes = if queued_eject { + emit(&both_full) + } else { + emit(&added_full) + }; + let batch_full = Raw(&batch_full_bytes); + let resume_later = Store::new(&has_job, &One {}); + let batch_exit = If::new(&batch_full, vec![&resume_later, &Break]); + let load_uid = Store::new(&uid, &cdat); + + let mark_event = Store::new(&has_event, &One {}); + let added_slot = Index::new(&Zero {}, &added_list, &num_added); + let cache_added = Store::new(&added_slot, &uid); + let count_added = Increment::new(&num_added); + let inserted = Equal::new(&cins, &One {}); + let on_insert = If::new(&inserted, vec![&cache_added, &count_added, &mark_event]); + + let eject_slot = Index::new(&Zero {}, &eject_list, &num_eject); + let cache_eject = Store::new(&eject_slot, &uid); + let count_eject = Increment::new(&num_eject); + let removed = Equal::new(&crmv, &One {}); + let queued_remove = If::new(&removed, vec![&cache_eject, &count_eject, &mark_event]); + let notify_removed = MethodCall::new(Path::new("CTFY"), vec![&uid, &EJECT_REQUEST]); + let clear_removed = Store::new(&crmv, &One {}); + let immediate_remove = If::new(&removed, vec![¬ify_removed, &clear_removed, &mark_event]); + let immediate_remove = Else::new(vec![&immediate_remove]); + let on_remove_bytes = if queued_eject { + emit(&queued_remove) + } else { + emit(&immediate_remove) + }; + let on_remove = Raw(&on_remove_bytes); + + let next_uid = Increment::new(&uid); + let pending = Equal::new(&has_event, &One {}); + let in_range = LessThan::new(&uid, &cpu_count); + let scanning = LAnd::new(&pending, &in_range); + let scan_loop = While::new( + &scanning, + vec![ + &clear_event, + &select, + &next_cpu, + &wrap_exit, + &batch_exit, + &load_uid, + &on_insert, + &on_remove, + &next_uid, + ], + ); + + // Notify OSPM about the collected CPUs and clear the events that got + // them onto the lists. + let first_idx = Store::new(&cpu_idx, &Zero {}); + let next_idx = Increment::new(&cpu_idx); + + let zero = Zero {}; + let added_slot_by_idx = Index::new(&zero, &added_list, &cpu_idx); + let added_elem = DeRefOf::new(&added_slot_by_idx); + let take_added = Store::new(&uid, &added_elem); + let notify_added = MethodCall::new(Path::new("CTFY"), vec![&uid, &DEVICE_CHECK]); + let trace = Store::new(&Debug, &uid); + let select_added = Store::new(&csel, &uid); + let clear_insert = Store::new(&cins, &One {}); + let more_added = LessThan::new(&cpu_idx, &num_added); + let added_loop = While::new( + &more_added, + vec![ + &take_added, + ¬ify_added, + &trace, + &select_added, + &clear_insert, + &next_idx, + ], + ); + + let eject_slot_by_idx = Index::new(&zero, &eject_list, &cpu_idx); + let eject_elem = DeRefOf::new(&eject_slot_by_idx); + let take_eject = Store::new(&uid, &eject_elem); + let notify_eject = MethodCall::new(Path::new("CTFY"), vec![&uid, &EJECT_REQUEST]); + let select_eject = Store::new(&csel, &uid); + let clear_remove = Store::new(&crmv, &One {}); + let more_eject = LessThan::new(&cpu_idx, &num_eject); + let eject_loop_term = While::new( + &more_eject, + vec![ + &take_eject, + ¬ify_eject, + &select_eject, + &clear_remove, + &next_idx, + ], + ); + let eject_loop_bytes = queued_eject.then(|| emit(&eject_loop_term)); + let eject_loop = Raw(match eject_loop_bytes.as_deref() { + Some(bytes) => bytes, + None => &[], + }); + let reset_eject_index_bytes = queued_eject.then(|| emit(&first_idx)); + let reset_eject_index = Raw(match reset_eject_index_bytes.as_deref() { + Some(bytes) => bytes, + None => &[], + }); + + let clear_job = Store::new(&has_job, &Zero {}); + let arm_event = Store::new(&has_event, &One {}); + let reset_added = Store::new(&num_added, &Zero {}); + let reset_eject_term = Store::new(&num_eject, &Zero {}); + let reset_eject_bytes = queued_eject.then(|| emit(&reset_eject_term)); + let reset_eject = Raw(match reset_eject_bytes.as_deref() { + Some(bytes) => bytes, + None => &[], + }); + let batching = Equal::new(&has_job, &One {}); + let batch_loop = While::new( + &batching, + vec![ + &clear_job, + &arm_event, + &reset_added, + &reset_eject, + &scan_loop, + &first_idx, + &added_loop, + &reset_eject_index, + &eject_loop, + ], + ); + + let release = Release::new(res("CPLK")); + emit(&Method::new( + Path::new("CSCN"), + 0, + true, + vec![ + &acquire, + &declare_added, + &declare_eject, + &first_uid, + &arm_job, + &batch_loop, + &release, + ], + )) +} + +/// `Method (COST, 4)`: hand an `_OST` status report to the controller. +fn ost_method() -> Vec<u8> { + let (csel, ccmd, cdat) = (res("CSEL"), res("CCMD"), res("CDAT")); + let acquire = Acquire::new(res("CPLK"), 0xffff); + let select = Store::new(&csel, &Arg(0)); + let event_cmd = Store::new(&ccmd, &CMD_OST_EVENT); + let event = Store::new(&cdat, &Arg(1)); + let status_cmd = Store::new(&ccmd, &CMD_OST_STATUS); + let status = Store::new(&cdat, &Arg(2)); + let release = Release::new(res("CPLK")); + emit(&Method::new( + Path::new("COST"), + 4, + true, + vec![ + &acquire, + &select, + &event_cmd, + &event, + &status_cmd, + &status, + &release, + ], + )) +} + +/// `CPU_NAME_FMT`: the AML name of a CPU object. +fn cpu_name(index: u32) -> String { + format!("C{index:03X}") +} + +/// `Package (count) {}`: a package that reserves room for `count` elements +/// but declares none. `PackageBuilder` always derives the count from the +/// elements added to it, so it cannot express this. +struct FixedPackage(u8); + +impl Aml for FixedPackage { + fn to_aml_bytes(&self, sink: &mut dyn AmlSink) { + sink.byte(0x12); + sink.vec(&pkg_length(1)); + sink.byte(self.0); + } +} + +/// `Break` +struct Break; + +impl Aml for Break { + fn to_aml_bytes(&self, sink: &mut dyn AmlSink) { + sink.byte(0xa5); + } +} + +/// `Debug`, the store target that forwards to the debug object. +struct Debug; + +impl Aml for Debug { + fn to_aml_bytes(&self, sink: &mut dyn AmlSink) { + sink.byte(0x5b); + sink.byte(0x31); + } +} + +macro_rules! logical_op { + ($(#[$doc:meta])* $name:ident, $opcode:expr) => { + $(#[$doc])* + struct $name<'a> { + left: &'a dyn Aml, + right: &'a dyn Aml, + } + + impl<'a> $name<'a> { + fn new(left: &'a dyn Aml, right: &'a dyn Aml) -> Self { + Self { left, right } + } + } + + impl Aml for $name<'_> { + fn to_aml_bytes(&self, sink: &mut dyn AmlSink) { + sink.byte($opcode); + self.left.to_aml_bytes(sink); + self.right.to_aml_bytes(sink); + } + } + }; +} + +logical_op!( + /// `(left && right)` + LAnd, + 0x90 +); +logical_op!( + /// `(left || right)` + LOr, + 0x91 +); + +#[cfg(test)] +mod tests { + #[test] + fn matches_qemu() { + let generated = + super::build(1, false, false, true).unwrap_or_else(|error| panic!("{error}")); + super::super::fixture::assert_region(&generated, 6299, 7354); + } +} diff --git a/dstack/crates/qemu-acpi/src/dsdt/crs.rs b/dstack/crates/qemu-acpi/src/dsdt/crs.rs new file mode 100644 index 000000000..9a763c89c --- /dev/null +++ b/dstack/crates/qemu-acpi/src/dsdt/crs.rs @@ -0,0 +1,211 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! The PCI0 root bus resources: the windows the root bridge decodes, plus the +//! two container devices that reserve the ACPI IO ports out of them. +//! +//! The caller wraps this term sequence in `Scope (\_SB.PCI0)`. +//! +//! ```asl +//! Name (_CRS, ResourceTemplate () { +//! WordBusNumber (ResourceProducer, MinFixed, MaxFixed, PosDecode, +//! 0x0000, 0x0000, 0x00FF, 0x0000, 0x0100, ,, ) +//! IO (Decode16, 0x0CF8, 0x0CF8, 0x01, 0x08, ) +//! WordIO (ResourceProducer, MinFixed, MaxFixed, PosDecode, EntireRange, +//! 0x0000, 0x0000, 0x0CF7, 0x0000, 0x0CF8, ,, , TypeStatic, DenseTranslation) +//! WordIO (ResourceProducer, MinFixed, MaxFixed, PosDecode, EntireRange, +//! 0x0000, 0x0D00, 0xFFFF, 0x0000, 0xF300, ,, , TypeStatic, DenseTranslation) +//! DWordMemory (ResourceProducer, PosDecode, MinFixed, MaxFixed, Cacheable, ReadWrite, +//! 0x00000000, 0x000A0000, 0x000BFFFF, 0x00000000, 0x00020000, +//! ,, , AddressRangeMemory, TypeStatic) +//! DWordMemory (ResourceProducer, PosDecode, MinFixed, MaxFixed, NonCacheable, ReadWrite, +//! 0x00000000, 0x80000000, 0xDFFFFFFF, 0x00000000, 0x60000000, +//! ,, , AddressRangeMemory, TypeStatic) +//! DWordMemory (ResourceProducer, PosDecode, MinFixed, MaxFixed, NonCacheable, ReadWrite, +//! 0x00000000, 0xF0000000, 0xFEBFFFFF, 0x00000000, 0x0EC00000, +//! ,, , AddressRangeMemory, TypeStatic) +//! QWordMemory (ResourceProducer, PosDecode, MinFixed, MaxFixed, Cacheable, ReadWrite, +//! 0x0000000000000000, 0x0000380000000000, 0x00003807FFFFFFFF, +//! 0x0000000000000000, 0x0000000800000000, ,, , AddressRangeMemory, TypeStatic) +//! }) +//! Device (GPE0) { +//! Name (_HID, "PNP0A06") +//! Name (_UID, "GPE0 resources") +//! Name (_STA, 0x0B) +//! Name (_CRS, ResourceTemplate () { IO (Decode16, 0x0620, 0x0620, 0x01, 0x10, ) }) +//! } +//! Device (PHPR) { +//! Name (_HID, "PNP0A06") +//! Name (_UID, "PCI Hotplug resources") +//! Name (_STA, 0x0B) +//! Name (_CRS, ResourceTemplate () { IO (Decode16, 0x0CC0, 0x0CC0, 0x01, 0x18, ) }) +//! } +//! ``` + +use acpi_tables::aml::{ + AddressSpace, AddressSpaceCacheable, Device, Name, Path, ResourceTemplate, IO, +}; +use acpi_tables::{Aml, AmlSink}; + +use super::ops::emit_all; + +/// Bus numbers the root bridge claims. Q35 exposes one segment of 256. +const BUS_MIN: u16 = 0x0000; +pub(crate) const BUS_MAX: u16 = 0x00ff; + +/// The PCI configuration mechanism 1 ports (`CF8`/`CFC`), consumed by the +/// bridge itself rather than forwarded onto the bus. +const PCI_CONFIG_IO_BASE: u16 = 0x0cf8; +const PCI_CONFIG_IO_LEN: u8 = 0x08; + +/// Every fixed IO descriptor below is byte aligned. +const IO_ALIGN: u8 = 0x01; + +/// Port IO forwarded to the bus, split around the configuration ports above. +const IO_LOW_MIN: u16 = 0x0000; +const IO_LOW_MAX: u16 = 0x0cf7; +const IO_HIGH_MIN: u16 = 0x0d00; +const IO_HIGH_MAX: u16 = 0xffff; + +/// The legacy VGA framebuffer aperture. +const VGA_MEM_MIN: u32 = 0x000a_0000; +const VGA_MEM_MAX: u32 = 0x000b_ffff; + +/// The 32-bit PCI hole: RAM below 4G ends where this window starts, so a +/// larger guest moves `PCI32_MIN` down. +#[cfg(test)] +const PCI32_MIN: u32 = 0x8000_0000; +const PCI32_MAX: u32 = 0xdfff_ffff; + +/// The window above PCIe ECAM and below the local APIC page. +const MMIO32_MIN: u32 = 0xf000_0000; +const MMIO32_MAX: u32 = 0xfebf_ffff; + +/// The 64-bit PCI hole, placed just above the guest's addressable RAM. +const PCI64_MIN: u64 = 0x0000_3800_0000_0000; +const PCI64_MAX: u64 = 0x0000_3807_ffff_ffff; + +struct Pci64Window { + max: u64, + length: u64, +} + +impl Aml for Pci64Window { + fn to_aml_bytes(&self, sink: &mut dyn AmlSink) { + sink.byte(0x8a); + sink.word(43); + sink.byte(0); // memory range + sink.byte(0x0c); // MinFixed | MaxFixed + sink.byte(0x03); // cacheable, read/write + sink.qword(0); // granularity + sink.qword(PCI64_MIN); + sink.qword(self.max); + sink.qword(0); // translation + sink.qword(self.length); + } +} + +/// GPE0 block ports, reserved so no PCI device is assigned over them. +const GPE0_IO_BASE: u16 = 0x0620; +const GPE0_IO_LEN: u8 = 0x10; + +/// PCI hotplug register block ports, reserved for the same reason. +const PHPR_IO_BASE: u16 = 0x0cc0; +const PHPR_IO_LEN: u8 = 0x18; + +/// `Name (_HID, "PNP0A06")`: both reservations are generic containers. +const CONTAINER_HID: &str = "PNP0A06"; +/// `Name (_STA, 0x0B)`: present, enabled and functioning, but not shown in UI. +const STA_HIDDEN: u8 = 0x0b; + +/// One IO port reservation device: a container that exists only to take a +/// fixed port range out of the root bus window. +fn reservation(name: &str, description: &'static str, base: u16, length: u8) -> Vec<u8> { + let hid = Name::new(Path::new("_HID"), &CONTAINER_HID); + let uid = Name::new(Path::new("_UID"), &description); + let sta = Name::new(Path::new("_STA"), &STA_HIDDEN); + + let port = IO::new(base, base, IO_ALIGN, length); + let template = ResourceTemplate::new(vec![&port]); + let crs = Name::new(Path::new("_CRS"), &template); + + emit_all(&[&Device::new(Path::new(name), vec![&hid, &uid, &sta, &crs])]) +} + +pub(crate) fn build( + low_ram_end: u32, + pci_hole64_size: Option<u64>, + pci_hotplug: bool, + bus_max: u16, +) -> Vec<u8> { + let buses = AddressSpace::new_bus_number(BUS_MIN, bus_max); + let config = IO::new( + PCI_CONFIG_IO_BASE, + PCI_CONFIG_IO_BASE, + IO_ALIGN, + PCI_CONFIG_IO_LEN, + ); + let io_low = AddressSpace::new_io(IO_LOW_MIN, IO_LOW_MAX, None); + let io_high = AddressSpace::new_io(IO_HIGH_MIN, IO_HIGH_MAX, None); + let vga = AddressSpace::new_memory( + AddressSpaceCacheable::Cacheable, + true, + VGA_MEM_MIN, + VGA_MEM_MAX, + None, + ); + let pci32 = AddressSpace::new_memory( + AddressSpaceCacheable::NotCacheable, + true, + low_ram_end, + PCI32_MAX, + None, + ); + let mmio32 = AddressSpace::new_memory( + AddressSpaceCacheable::NotCacheable, + true, + MMIO32_MIN, + MMIO32_MAX, + None, + ); + let pci64_length = pci_hole64_size.unwrap_or(PCI64_MAX - PCI64_MIN + 1); + let pci64 = Pci64Window { + max: PCI64_MIN.wrapping_add(pci64_length).wrapping_sub(1), + length: pci64_length, + }; + + let template = ResourceTemplate::new(vec![ + &buses, &config, &io_low, &io_high, &vga, &pci32, &mmio32, &pci64, + ]); + let crs = Name::new(Path::new("_CRS"), &template); + + let mut out = emit_all(&[&crs]); + out.extend(reservation( + "GPE0", + "GPE0 resources", + GPE0_IO_BASE, + GPE0_IO_LEN, + )); + if pci_hotplug { + out.extend(reservation( + "PHPR", + "PCI Hotplug resources", + PHPR_IO_BASE, + PHPR_IO_LEN, + )); + } + out +} + +#[cfg(test)] +mod tests { + #[test] + fn matches_qemu() { + super::super::fixture::assert_region( + &super::build(super::PCI32_MIN, None, true, super::BUS_MAX), + 7395, + 7732, + ); + } +} diff --git a/dstack/crates/qemu-acpi/src/dsdt/dbg.rs b/dstack/crates/qemu-acpi/src/dsdt/dbg.rs new file mode 100644 index 000000000..15eac7602 --- /dev/null +++ b/dstack/crates/qemu-acpi/src/dsdt/dbg.rs @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! `Scope (\)`: the QEMU debug port and the `DBUG` string writer. +//! +//! ```asl +//! Scope (\) { +//! OperationRegion (DBG, SystemIO, 0x0402, One) +//! Field (DBG, ByteAcc, NoLock, Preserve) { DBGB, 8 } +//! Method (DBUG, 1, NotSerialized) { +//! ToHexString (Arg0, Local0) +//! ToBuffer (Local0, Local0) +//! Local1 = (SizeOf (Local0) - One) +//! Local2 = Zero +//! While ((Local2 < Local1)) { +//! DBGB = DerefOf (Local0 [Local2]) +//! Local2++ +//! } +//! DBGB = 0x0A +//! } +//! } +//! ``` + +use acpi_tables::aml::{ + Arg, DeRefOf, FieldAccessType, FieldEntry, FieldLockRule, FieldUpdateRule, Index, LessThan, + Local, One, OpRegion, OpRegionSpace, Path, SizeOf, Store, Subtract, ToBuffer, While, Zero, +}; + +use super::ops::{emit_all, root_scope, Increment, ToHexString}; + +pub(crate) fn build() -> Vec<u8> { + let region = OpRegion::new( + Path::new("DBG_"), + OpRegionSpace::SystemIO, + &0x402u16, + &One {}, + ); + let field = acpi_tables::aml::Field::new( + Path::new("DBG_"), + FieldAccessType::Byte, + FieldLockRule::NoLock, + FieldUpdateRule::Preserve, + vec![FieldEntry::Named(*b"DBGB", 8)], + ); + + let (local0, local1, local2) = (Local(0), Local(1), Local(2)); + let to_hex = ToHexString::new(&local0, &Arg(0)); + let to_buffer = ToBuffer::new(&local0, &local0); + let size = SizeOf::new(&local0); + let length = Subtract::new(&local1, &size, &One {}); + let init = Store::new(&local2, &Zero {}); + + let dbgb = Path::new("DBGB"); + let index = Index::new(&Zero {}, &local0, &local2); + let deref = DeRefOf::new(&index); + let write = Store::new(&dbgb, &deref); + let advance = Increment::new(&local2); + let condition = LessThan::new(&local2, &local1); + let loop_ = While::new(&condition, vec![&write, &advance]); + + let newline = Store::new(&dbgb, &0x0au8); + let method = acpi_tables::aml::Method::new( + Path::new("DBUG"), + 1, + false, + vec![&to_hex, &to_buffer, &length, &init, &loop_, &newline], + ); + + root_scope(&emit_all(&[®ion, &field, &method])) +} + +#[cfg(test)] +mod tests { + #[test] + fn matches_qemu() { + super::super::fixture::assert_region(&super::build(), 36, 110); + } +} diff --git a/dstack/crates/qemu-acpi/src/dsdt/fwcf.rs b/dstack/crates/qemu-acpi/src/dsdt/fwcf.rs new file mode 100644 index 000000000..04e60dc63 --- /dev/null +++ b/dstack/crates/qemu-acpi/src/dsdt/fwcf.rs @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! `Scope (\_SB.PCI0)`: the fw_cfg device, so the guest can find the fw_cfg +//! I/O ports without probing them. +//! +//! ```asl +//! Scope (\_SB.PCI0) { +//! Device (FWCF) { +//! Name (_HID, "QEMU0002") +//! Name (_STA, 0x0B) +//! Name (_CRS, ResourceTemplate () { +//! IO (Decode16, 0x0510, 0x0510, 0x01, 0x0C) +//! }) +//! } +//! } +//! ``` + +use acpi_tables::aml::{AmlStr, Device, Name, Path, ResourceTemplate, Scope, IO}; + +use super::ops::emit; + +pub(crate) fn build() -> Vec<u8> { + let id: AmlStr = "QEMU0002"; + let hid = Name::new(Path::new("_HID"), &id); + // Present and functioning, but hidden from the user interface. + let sta = Name::new(Path::new("_STA"), &0x0bu8); + + // The fw_cfg selector, data and DMA registers at 0x510..0x51c. + let ports = IO::new(0x510, 0x510, 0x01, 0x0c); + let template = ResourceTemplate::new(vec![&ports]); + let crs = Name::new(Path::new("_CRS"), &template); + + let device = Device::new(Path::new("FWCF"), vec![&hid, &sta, &crs]); + Scope::raw(Path::new("\\_SB_.PCI0"), emit(&device)) +} + +#[cfg(test)] +mod tests { + #[test] + fn matches_qemu() { + super::super::fixture::assert_region(&super::build(), 7774, 7834); + } +} diff --git a/dstack/crates/qemu-acpi/src/dsdt/gpe.rs b/dstack/crates/qemu-acpi/src/dsdt/gpe.rs new file mode 100644 index 000000000..46342124b --- /dev/null +++ b/dstack/crates/qemu-acpi/src/dsdt/gpe.rs @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! The general purpose event block: its device ID, and the two edge-triggered +//! handlers QEMU wires up. These three fragments are far apart in the table, +//! so each is built and verified separately. +//! +//! ```asl +//! Scope (_GPE) { +//! Name (_HID, "ACPI0006") +//! } +//! +//! Method (\_GPE._E02, 0, NotSerialized) { +//! \_SB.CPUS.CSCN () +//! } +//! +//! Scope (_GPE) { +//! Method (_E01, 0, NotSerialized) { +//! } +//! } +//! ``` +//! +//! `_E01` is deliberately empty here: it is the PCI hotplug event, and the +//! baseline machine has no hotplug-capable bridge for it to scan. + +use acpi_tables::aml::{Method, MethodCall, Name, Path, Scope}; + +use super::ops::emit_all; + +/// `Name (_HID, "ACPI0006")`: the GPE block device ID. +const GPE_BLOCK_HID: &str = "ACPI0006"; + +/// `Scope (_GPE) { Name (_HID, "ACPI0006") }` +pub(crate) fn hid() -> Vec<u8> { + let id = Name::new(Path::new("_HID"), &GPE_BLOCK_HID); + emit_all(&[&Scope::new(Path::new("_GPE"), vec![&id])]) +} + +/// `Method (\_GPE._E02, 0, NotSerialized) { \_SB.CPUS.CSCN () }`, the CPU +/// hotplug event: rescan the CPU devices. +pub(crate) fn e02() -> Vec<u8> { + let scan = MethodCall::new(Path::new("\\_SB_.CPUS.CSCN"), vec![]); + emit_all(&[&Method::new( + Path::new("\\_GPE._E02"), + 0, + false, + vec![&scan], + )]) +} + +/// `Scope (_GPE) { Method (_E01, 0, NotSerialized) {} }`, the PCI hotplug +/// event. +pub(crate) fn e01() -> Vec<u8> { + let handler = Method::new(Path::new("_E01"), 0, false, vec![]); + emit_all(&[&Scope::new(Path::new("_GPE"), vec![&handler])]) +} + +#[cfg(test)] +mod tests { + #[test] + fn hid_matches_qemu() { + super::super::fixture::assert_region(&super::hid(), 6271, 6292); + } + + #[test] + fn e02_matches_qemu() { + super::super::fixture::assert_region(&super::e02(), 7354, 7382); + } + + #[test] + fn e01_matches_qemu() { + super::super::fixture::assert_region(&super::e01(), 8245, 8258); + } +} diff --git a/dstack/crates/qemu-acpi/src/dsdt/links.rs b/dstack/crates/qemu-acpi/src/dsdt/links.rs new file mode 100644 index 000000000..e55e9c527 --- /dev/null +++ b/dstack/crates/qemu-acpi/src/dsdt/links.rs @@ -0,0 +1,264 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! PCI interrupt link devices: the two shared helpers plus sixteen devices. +//! +//! `IQST`/`IQCR` decode a PCI interrupt route register (the `PRQx` bytes the +//! LPC device exposes as a field): bit 7 means "disabled", the low nibble is +//! the routed ISA IRQ. +//! +//! ```asl +//! Method (IQST, 1, NotSerialized) { +//! If ((0x80 & Arg0)) { Return (0x09) } // present, not enabled +//! Return (0x0B) // present and enabled +//! } +//! Method (IQCR, 1, Serialized) { +//! Name (PRR0, ResourceTemplate () { +//! Interrupt (ResourceConsumer, Level, ActiveHigh, Shared, ,, _Y00) { 0 } +//! }) +//! CreateDWordField (PRR0, \_SB.IQCR._Y00._INT, PRRI) +//! PRRI = (Arg0 & 0x0F) +//! Return (PRR0) +//! } +//! ``` +//! +//! `LNKA`..`LNKH` are the eight routable links, one per PIRQ. They differ only +//! in device name, `_UID` (0..7) and the route register they read and write +//! (`PRQA`..`PRQH`); the ASL below is `LNKA` with those three holes. +//! +//! ```asl +//! Device (LNKA) { +//! Name (_HID, EisaId ("PNP0C0F")) +//! Name (_UID, Zero) +//! Name (_PRS, ResourceTemplate () { +//! Interrupt (ResourceConsumer, Level, ActiveHigh, Shared, ,, ) { 5, 10, 11 } +//! }) +//! Method (_STA, 0, NotSerialized) { Return (IQST (PRQA)) } +//! Method (_DIS, 0, NotSerialized) { PRQA |= 0x80 } +//! Method (_CRS, 0, NotSerialized) { Return (IQCR (PRQA)) } +//! Method (_SRS, 1, NotSerialized) { +//! CreateDWordField (Arg0, 0x05, PRRI) +//! PRQA = PRRI +//! } +//! } +//! ``` +//! +//! `GSIA`..`GSIH` are the fixed IOAPIC GSIs 16..23 that the same PIRQs land on +//! when the chipset is not in PIC mode. They are not routable, so `_CRS` is a +//! constant and `_DIS`/`_SRS` are empty. They differ only in device name and in +//! the single number that is both `_UID` and the interrupt: `0x10 + index`. +//! +//! ```asl +//! Device (GSIA) { +//! Name (_HID, EisaId ("PNP0C0F")) +//! Name (_UID, 0x10) +//! Name (_PRS, ResourceTemplate () { +//! Interrupt (ResourceConsumer, Level, ActiveHigh, Shared, ,, ) { 0x10 } +//! }) +//! Name (_CRS, ResourceTemplate () { +//! Interrupt (ResourceConsumer, Level, ActiveHigh, Shared, ,, ) { 0x10 } +//! }) +//! Method (_DIS, 0, NotSerialized) { } +//! Method (_SRS, 1, NotSerialized) { } +//! } +//! ``` + +use acpi_tables::aml::{ + And, Arg, CreateDWordField, Device, EISAName, If, Interrupt, Method, MethodCall, Name, Or, + Path, ResourceTemplate, Return, Store, Zero, +}; +use acpi_tables::{Aml, AmlSink}; + +use super::ops::emit; + +/// PNP ID shared by every link device, routable or not. +const LINK_HID: &str = "PNP0C0F"; +/// The suffix letters, in the order QEMU emits the devices. +const LETTERS: [char; 8] = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H']; +/// ISA IRQs a PIRQ may be routed to. QEMU offers the same three for every link. +const LINK_IRQS: [u32; 3] = [5, 10, 11]; +/// GSI of the first non-routable link; the rest follow consecutively. +const GSI_BASE: u8 = 0x10; +/// `_STA` bit set when a device is present but not enabled. +const STA_PRESENT: u8 = 0x09; +/// `_STA` bits set when a device is present and enabled. +const STA_ENABLED: u8 = 0x0b; +/// Route register bit meaning "this PIRQ is masked". +const ROUTE_DISABLE: u8 = 0x80; +/// Route register mask selecting the routed ISA IRQ. +const ROUTE_IRQ: u8 = 0x0f; +/// Byte offset of the interrupt number inside an extended IRQ descriptor, as +/// seen from the start of a single-descriptor resource template buffer. +const DESCRIPTOR_INT_OFFSET: u8 = 0x05; + +/// Extended interrupt descriptor listing several interrupt numbers. +/// `acpi_tables::aml::Interrupt` only models the single-number form, which is +/// all a `_CRS` ever needs, but a `_PRS` enumerates every possible routing. +struct InterruptList<'a> { + consumer: bool, + edge_triggered: bool, + active_low: bool, + shared: bool, + numbers: &'a [u32], +} + +impl<'a> InterruptList<'a> { + fn new( + consumer: bool, + edge_triggered: bool, + active_low: bool, + shared: bool, + numbers: &'a [u32], + ) -> Self { + Self { + consumer, + edge_triggered, + active_low, + shared, + numbers, + } + } +} + +impl Aml for InterruptList<'_> { + fn to_aml_bytes(&self, sink: &mut dyn AmlSink) { + sink.byte(0x89); /* Extended IRQ Descriptor */ + sink.word(2 + 4 * self.numbers.len() as u16); + let flags = ((self.shared as u8) << 3) + | ((self.active_low as u8) << 2) + | ((self.edge_triggered as u8) << 1) + | self.consumer as u8; + sink.byte(flags); + sink.byte(self.numbers.len() as u8); + for number in self.numbers { + sink.dword(*number); + } + } +} + +pub(crate) fn build(edge_triggered: bool) -> Vec<u8> { + let mut out = interrupt_status(); + out.extend(interrupt_resource(edge_triggered)); + for (index, letter) in LETTERS.iter().enumerate() { + out.extend(link_device(*letter, index as u8, edge_triggered)); + } + for (index, letter) in LETTERS.iter().enumerate() { + out.extend(gsi_device(*letter, GSI_BASE + index as u8, edge_triggered)); + } + out +} + +/// `Method (IQST, 1)`: turn a route register value into a `_STA` result. +fn interrupt_status() -> Vec<u8> { + let masked = And::new(&Zero {}, &ROUTE_DISABLE, &Arg(0)); + let present = Return::new(&STA_PRESENT); + let disabled = If::new(&masked, vec![&present]); + let enabled = Return::new(&STA_ENABLED); + emit(&Method::new( + Path::new("IQST"), + 1, + false, + vec![&disabled, &enabled], + )) +} + +/// `Method (IQCR, 1)`: turn a route register value into a `_CRS` buffer. +/// Serialized because it patches the interrupt number into a named buffer. +fn interrupt_resource(edge_triggered: bool) -> Vec<u8> { + let template = Interrupt::new(true, edge_triggered, false, true, 0); + let buffer = ResourceTemplate::new(vec![&template]); + let named = Name::new(Path::new("PRR0"), &buffer); + + let source = Path::new("PRR0"); + let field = Path::new("PRRI"); + let overlay = CreateDWordField::new(&field, &source, &DESCRIPTOR_INT_OFFSET); + + let routed = And::new(&Zero {}, &Arg(0), &ROUTE_IRQ); + let target = Path::new("PRRI"); + let patch = Store::new(&target, &routed); + + let result = Path::new("PRR0"); + let ret = Return::new(&result); + + emit(&Method::new( + Path::new("IQCR"), + 1, + true, + vec![&named, &overlay, &patch, &ret], + )) +} + +/// One routable link: `Device (LNK<letter>)` over route register `PRQ<letter>`. +fn link_device(letter: char, uid: u8, edge_triggered: bool) -> Vec<u8> { + let hid = Name::new(Path::new("_HID"), &EISAName::new(LINK_HID)); + let unique = Name::new(Path::new("_UID"), &uid); + + let possible = InterruptList::new(true, edge_triggered, false, true, &LINK_IRQS); + let buffer = ResourceTemplate::new(vec![&possible]); + let prs = Name::new(Path::new("_PRS"), &buffer); + + let register = format!("PRQ{letter}"); + + // _STA: Return (IQST (PRQx)) + let status_arg = Path::new(®ister); + let status_call = MethodCall::new(Path::new("IQST"), vec![&status_arg]); + let status_ret = Return::new(&status_call); + let sta = Method::new(Path::new("_STA"), 0, false, vec![&status_ret]); + + // _DIS: PRQx |= 0x80 + let disable_target = Path::new(®ister); + let disable_source = Path::new(®ister); + let mask = Or::new(&disable_target, &disable_source, &ROUTE_DISABLE); + let dis = Method::new(Path::new("_DIS"), 0, false, vec![&mask]); + + // _CRS: Return (IQCR (PRQx)) + let current_arg = Path::new(®ister); + let current_call = MethodCall::new(Path::new("IQCR"), vec![¤t_arg]); + let current_ret = Return::new(¤t_call); + let crs = Method::new(Path::new("_CRS"), 0, false, vec![¤t_ret]); + + // _SRS: CreateDWordField (Arg0, 0x05, PRRI); PRQx = PRRI + let field = Path::new("PRRI"); + let overlay = CreateDWordField::new(&field, &Arg(0), &DESCRIPTOR_INT_OFFSET); + let store_target = Path::new(®ister); + let store_source = Path::new("PRRI"); + let apply = Store::new(&store_target, &store_source); + let srs = Method::new(Path::new("_SRS"), 1, false, vec![&overlay, &apply]); + + let name = format!("LNK{letter}"); + emit(&Device::new( + Path::new(&name), + vec![&hid, &unique, &prs, &sta, &dis, &crs, &srs], + )) +} + +/// One fixed link: `Device (GSI<letter>)` pinned to interrupt `gsi`. +fn gsi_device(letter: char, gsi: u8, edge_triggered: bool) -> Vec<u8> { + let hid = Name::new(Path::new("_HID"), &EISAName::new(LINK_HID)); + let unique = Name::new(Path::new("_UID"), &gsi); + + let interrupt = Interrupt::new(true, edge_triggered, false, true, u32::from(gsi)); + let possible = ResourceTemplate::new(vec![&interrupt]); + let prs = Name::new(Path::new("_PRS"), &possible); + let current = ResourceTemplate::new(vec![&interrupt]); + let crs = Name::new(Path::new("_CRS"), ¤t); + + let empty: Vec<&dyn Aml> = Vec::new(); + let dis = Method::new(Path::new("_DIS"), 0, false, empty.clone()); + let srs = Method::new(Path::new("_SRS"), 1, false, empty); + + let name = format!("GSI{letter}"); + emit(&Device::new( + Path::new(&name), + vec![&hid, &unique, &prs, &crs, &dis, &srs], + )) +} + +#[cfg(test)] +mod tests { + #[test] + fn matches_qemu() { + super::super::fixture::assert_region(&super::build(false), 4552, 6271); + } +} diff --git a/dstack/crates/qemu-acpi/src/dsdt/mod.rs b/dstack/crates/qemu-acpi/src/dsdt/mod.rs new file mode 100644 index 000000000..4f33fb9de --- /dev/null +++ b/dstack/crates/qemu-acpi/src/dsdt/mod.rs @@ -0,0 +1,235 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! From-scratch generation of QEMU's Q35 DSDT. +//! +//! The DSDT is the only ACPI table QEMU builds out of AML rather than fixed +//! records, and it is 93% of `etc/acpi/tables`. Every term here is emitted in +//! the same order QEMU's `build_dsdt()` emits it, because the byte stream is +//! the measured artifact: an equivalent-but-differently-ordered DSDT would be +//! a different RTMR0. +//! +//! Each submodule owns one contiguous byte range of the table and is verified +//! against the captured QEMU fixture on its own, so a mismatch points at a +//! region instead of at the whole blob. + +use crate::{Compatibility, Error, MachineConfig}; +use acpi_tables::aml::{Path, Scope}; + +mod cpus; +mod crs; +mod dbg; +mod fwcf; +mod gpe; +mod links; +mod notify; +mod ops; +mod pci0; +mod pcihp; +mod pic; +mod prt; +mod pxb; +mod sstate; + +/// DSDT body: everything after the 36-byte ACPI table header. +pub(crate) fn body(config: &MachineConfig) -> Result<Vec<u8>, Error> { + let pci_hotplug = !config.hotplug_off; + let has_pxb = config.hugepages && config.num_gpus > 0; + let low_ram_end = if config.memory_size >= 0xb000_0000 { + 0x8000_0000 + } else { + config.memory_size as u32 + }; + let fixed_slots = 4 + u32::from(config.root_verity); + let regular_slots = fixed_slots + config.num_nics + config.num_verity_volumes; + let root_ports = if config.hugepages && config.num_gpus > 0 { + config.num_nvswitches + } else { + config.num_gpus + config.num_nvswitches + }; + let modern_serial_irq = matches!( + config.qemu_version.compatibility(), + Some(Compatibility::V11_1) + ); + let initialize_cpu_selector = matches!( + config.qemu_version.compatibility(), + Some(Compatibility::V8 | Compatibility::V9Pre92 | Compatibility::V9_2 | Compatibility::V10) + ); + let queued_cpu_eject = matches!( + config.qemu_version.compatibility(), + Some(Compatibility::V10 | Compatibility::V11_0 | Compatibility::V11_1) + ); + let modern_device_label = !matches!( + config.qemu_version.compatibility(), + Some(Compatibility::V8 | Compatibility::V9Pre92) + ); + let edge_triggered_links = !modern_device_label; + let mut out = Vec::new(); + out.extend(dbg::build()); // 36..110 Scope(\) debug port + out.extend(pci0::build(pci_hotplug)); // 110..426 Scope(_SB) PCI0 + DRAC + if pci_hotplug { + out.extend(pcihp::build(modern_device_label)); + } + out.extend(pic::build()); // 777..796 PICF + _PIC + out.extend(Scope::raw( + Path::new("_SB_"), + [prt::build(), links::build(edge_triggered_links)].concat(), + )); // 796..6271 Scope(_SB) routing + links + out.extend(gpe::hid()); // 6271..6292 Scope(_GPE) _HID + out.extend(Scope::raw( + Path::new("_SB_"), + cpus::build( + config.cpu_count, + config.hugepages, + initialize_cpu_selector, + queued_cpu_eject, + )?, + )); + out.extend(gpe::e02()); // 7354..7382 \_GPE._E02 + if has_pxb { + out.extend(pxb::build()); + } + out.extend(Scope::raw( + Path::new("\\_SB_.PCI0"), + crs::build( + low_ram_end, + config.pci_hole64_size, + pci_hotplug, + if has_pxb { 4 } else { crs::BUS_MAX }, + ), + )); + out.extend(sstate::build()); // 7732..7774 Scope(\) _S3/_S4/_S5 + out.extend(fwcf::build()); // 7774..7834 Scope(\_SB.PCI0) FWCF + out.extend(notify::build( + regular_slots, + root_ports, + modern_serial_irq, + has_pxb.then_some(0x80), + )); + if pci_hotplug { + out.extend(gpe::e01()); + } + Ok(out) +} + +/// Complete DSDT, including its standard ACPI header. +pub(crate) fn table(config: &MachineConfig) -> Result<Vec<u8>, Error> { + let body = body(config)?; + let length = 36 + body.len(); + let mut out = Vec::with_capacity(length); + out.extend_from_slice(b"DSDT"); + out.extend_from_slice(&(length as u32).to_le_bytes()); + out.extend_from_slice(&[1, 0]); // revision, checksum (fixed by firmware) + out.extend_from_slice(b"BOCHS "); + out.extend_from_slice(b"BXPC "); + out.extend_from_slice(&1u32.to_le_bytes()); + out.extend_from_slice(b"BXPC"); + out.extend_from_slice(&1u32.to_le_bytes()); + out.extend(body); + Ok(out) +} + +#[cfg(test)] +pub(crate) mod fixture { + /// Captured QEMU output for the baseline machine: 1 vCPU, no extra PCI + /// devices, CPU hotplug on, no NUMA/PXB. It is a test oracle only and is + /// never part of generation. + const BASE: &[u8] = include_bytes!("../../fixtures/qemu-11.1-q35-base.bin"); + const DSDT_OFFSET: usize = 64; + const DSDT_LEN: usize = 8258; + /// Length of the ACPI table header preceding the AML body. + pub(crate) const HEADER_LEN: usize = 36; + + pub(crate) fn dsdt() -> &'static [u8] { + &BASE[DSDT_OFFSET..DSDT_OFFSET + DSDT_LEN] + } + + /// The whole captured `etc/acpi/tables` blob, trimmed of its trailing + /// zero padding. Offsets are the ones the loader relocates against. + pub(crate) fn base() -> &'static [u8] { + BASE + } + + /// Compare generated bytes against a byte range of the whole blob. + pub(crate) fn assert_blob_range(actual: &[u8], start: usize, end: usize) { + assert_slice(actual, &base()[start..end], start); + } + + /// Compare generated bytes against a DSDT byte range, reporting the first + /// divergence with enough context to find it in an `iasl -d` listing. + pub(crate) fn assert_region(actual: &[u8], start: usize, end: usize) { + assert_slice(actual, &dsdt()[start..end], start); + } + + fn assert_slice(actual: &[u8], expected: &[u8], start: usize) { + if actual == expected { + return; + } + let common = actual + .iter() + .zip(expected) + .take_while(|(a, b)| a == b) + .count(); + let window = |bytes: &[u8], from: usize| { + bytes + .iter() + .skip(from) + .take(16) + .map(|b| format!("{b:02x}")) + .collect::<Vec<_>>() + .join(" ") + }; + panic!( + "region at offset {start} does not match QEMU\n\ + matched {common} of {} bytes (generated {} bytes)\n\ + first difference at offset {}\n\ + expected: {}\n\ + actual: {}", + expected.len(), + actual.len(), + start + common, + window(expected, common), + window(actual, common), + ); + } +} + +#[cfg(test)] +mod tests { + use super::fixture; + use crate::{MachineConfig, QemuVersion}; + + fn config() -> MachineConfig { + MachineConfig { + qemu_version: QemuVersion::new(11, 1, 0), + cpu_count: 1, + memory_size: 2 << 30, + pic: false, + smm: false, + hugepages: false, + num_gpus: 0, + num_nvswitches: 0, + num_nics: 0, + num_verity_volumes: 0, + hotplug_off: false, + root_verity: true, + pci_hole64_size: None, + } + } + + /// The whole body, once every region lands. Until then the per-region + /// tests are the ones that matter; this is the integration check that no + /// term is missing, duplicated, or emitted out of order. + #[test] + fn body_matches_qemu() { + let generated = super::body(&config()).unwrap_or_else(|error| panic!("{error}")); + fixture::assert_region(&generated, fixture::HEADER_LEN, fixture::dsdt().len()); + } + + #[test] + fn table_matches_qemu() { + let generated = super::table(&config()).unwrap_or_else(|error| panic!("{error}")); + assert_eq!(generated, fixture::dsdt()); + } +} diff --git a/dstack/crates/qemu-acpi/src/dsdt/notify.rs b/dstack/crates/qemu-acpi/src/dsdt/notify.rs new file mode 100644 index 000000000..2039a5bfc --- /dev/null +++ b/dstack/crates/qemu-acpi/src/dsdt/notify.rs @@ -0,0 +1,303 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! `Scope (\_SB) { Scope (PCI0) { .. } }`: one descriptor per populated slot on +//! the root bus, and the ISA scaffolding hanging off the ICH9 LPC bridge. +//! +//! QEMU walks every devfn of the bus (`build_append_pci_bus_devices`, +//! `hw/acpi/pcihp.c`) and emits `Device (S<devfn>) { Name (_ADR, ..) }` for the +//! populated ones, then lets each device append its own AML. Only the LPC +//! bridge does (`build_ich9_isa_aml`, `hw/isa/lpc_ich9.c`). +//! +//! ```asl +//! Scope (\_SB) { +//! Scope (PCI0) { +//! Device (S00) { Name (_ADR, Zero) } +//! Device (S08) { Name (_ADR, 0x00010000) } +//! Device (S10) { Name (_ADR, 0x00020000) } +//! Device (S18) { Name (_ADR, 0x00030000) } +//! Device (S20) { Name (_ADR, 0x00040000) } +//! Device (SF8) { +//! Name (_ADR, 0x001F0000) +//! OperationRegion (PIRQ, PCI_Config, 0x60, 0x0C) +//! Scope (\_SB) { +//! Field (PCI0.SF8.PIRQ, ByteAcc, NoLock, Preserve) { +//! PRQA, 8, PRQB, 8, PRQC, 8, PRQD, 8, +//! Offset (0x08), +//! PRQE, 8, PRQF, 8, PRQG, 8, PRQH, 8 +//! } +//! } +//! Device (KBD) { +//! Name (_HID, EisaId ("PNP0303")) +//! Name (_STA, 0x0F) +//! Name (_CRS, ResourceTemplate () { +//! IO (Decode16, 0x0060, 0x0060, 0x01, 0x01) +//! IO (Decode16, 0x0064, 0x0064, 0x01, 0x01) +//! IRQNoFlags () {1} +//! }) +//! } +//! Device (MOU) { +//! Name (_HID, EisaId ("PNP0F13")) +//! Name (_STA, 0x0F) +//! Name (_CRS, ResourceTemplate () { IRQNoFlags () {12} }) +//! } +//! Device (COM1) { +//! Name (_HID, EisaId ("PNP0501")) +//! Name (_UID, One) +//! Name (_STA, 0x0F) +//! Name (_CRS, ResourceTemplate () { +//! IO (Decode16, 0x03F8, 0x03F8, 0x00, 0x08) +//! IRQ (Level, ActiveLow, Shared, ) {4} +//! }) +//! } +//! Device (RTC) { +//! Name (_HID, EisaId ("PNP0B00")) +//! Name (_CRS, ResourceTemplate () { +//! IO (Decode16, 0x0070, 0x0070, 0x01, 0x08) +//! IRQNoFlags () {8} +//! }) +//! } +//! } +//! Device (SFA) { Name (_ADR, 0x001F0002) } +//! Device (SFB) { Name (_ADR, 0x001F0003) } +//! } +//! } +//! ``` + +use acpi_tables::aml::{ + Device, EISAName, Field, FieldAccessType, FieldEntry, FieldLockRule, FieldUpdateRule, Name, + OpRegion, OpRegionSpace, Path, ResourceTemplate, Scope, IO, +}; +use acpi_tables::{Aml, AmlSink}; + +use super::ops::{emit, Raw}; + +/// Devfns populated on the root bus of the baseline machine by the ICH9 chipset: +/// LPC (which emits extra AML), SATA, and SMBus. +const CHIPSET_DEVFNS: &[u8] = &[0xf8, 0xfa, 0xfb]; + +/// The ICH9 LPC bridge, the only function that contributes its own AML. +const LPC_DEVFN: u8 = 0xf8; + +pub(crate) fn build( + slot_count: u32, + root_port_count: u32, + modern_serial_irq: bool, + pxb_devfn: Option<u8>, +) -> Vec<u8> { + let lpc = lpc_children(modern_serial_irq); + + let mut devices = Vec::new(); + for slot in 0..slot_count { + devices.push(((slot * 8) as u8, false)); + } + if let Some(devfn) = pxb_devfn { + devices.push((devfn, false)); + } + let mut slot = slot_count; + for _ in 0..root_port_count { + if pxb_devfn == Some((slot * 8) as u8) { + slot += 1; + } + devices.push(((slot * 8) as u8, true)); + slot += 1; + } + for &devfn in CHIPSET_DEVFNS { + devices.push((devfn, false)); + } + devices.sort_unstable_by_key(|(devfn, _)| *devfn); + + let mut bus = Vec::new(); + for (devfn, root_port) in devices { + bus.extend(pci_device(devfn, root_port, &lpc)); + } + + Scope::raw(Path::new("\\_SB_"), Scope::raw(Path::new("PCI0"), bus)) +} + +fn pci_device(devfn: u8, root_port: bool, lpc: &[u8]) -> Vec<u8> { + // QEMU names the device after the devfn but addresses it by the + // ACPI 1.0b Table 6-2 PCI form: (device << 16) | function. + let name = format!("S{devfn:02X}_"); + let address = (u32::from(devfn >> 3) << 16) | u32::from(devfn & 0x07); + let adr = Name::new(Path::new("_ADR"), &address); + + let child_address = Name::new(Path::new("_ADR"), &0u8); + let child = root_port.then(|| emit(&Device::new(Path::new("S00_"), vec![&child_address]))); + let extra = Raw(if devfn == LPC_DEVFN { lpc } else { &[] }); + let child = Raw(match child.as_deref() { + Some(bytes) => bytes, + None => &[], + }); + emit(&Device::new(Path::new(&name), vec![&adr, &extra, &child])) +} + +/// The children the ICH9 LPC bridge appends to its own device descriptor. +fn lpc_children(modern_serial_irq: bool) -> Vec<u8> { + // PCI-to-ISA interrupt routing registers in the bridge's config space. + let pirq = OpRegion::new( + Path::new("PIRQ"), + OpRegionSpace::PCIConfig, + &0x60u8, + &0x0cu8, + ); + + // The field lands in \_SB rather than in the device, because the link + // devices that read PRQA..PRQH live there. It has to follow the operation + // region it names. + let routing = Field::new( + Path::new("PCI0.SF8_.PIRQ"), + FieldAccessType::Byte, + FieldLockRule::NoLock, + FieldUpdateRule::Preserve, + vec![ + FieldEntry::Named(*b"PRQA", 8), + FieldEntry::Named(*b"PRQB", 8), + FieldEntry::Named(*b"PRQC", 8), + FieldEntry::Named(*b"PRQD", 8), + // Offset (0x08): PIRQE..PIRQH sit at 0x68, four bytes on. + FieldEntry::Reserved(0x20), + FieldEntry::Named(*b"PRQE", 8), + FieldEntry::Named(*b"PRQF", 8), + FieldEntry::Named(*b"PRQG", 8), + FieldEntry::Named(*b"PRQH", 8), + ], + ); + + let mut out = emit(&pirq); + out.extend(Scope::raw(Path::new("\\_SB_"), emit(&routing))); + out.extend(isa_devices(modern_serial_irq)); + out +} + +/// The devices on the bridge's ISA bus, in qbus order. +fn isa_devices(modern_serial_irq: bool) -> Vec<u8> { + let mut out = Vec::new(); + + // i8042: data port, command port, keyboard IRQ (hw/input/pckbd.c). + let kbd_data = IO::new(0x60, 0x60, 0x01, 0x01); + let kbd_command = IO::new(0x64, 0x64, 0x01, 0x01); + let kbd_irq = Irq::no_flags(1); + out.extend(isa_device( + "KBD_", + "PNP0303", + None, + Some(0x0f), + vec![&kbd_data, &kbd_command, &kbd_irq], + )); + + // The i8042's mouse half is a separate ACPI device sharing the ports. + let mouse_irq = Irq::no_flags(12); + out.extend(isa_device( + "MOU_", + "PNP0F13", + None, + Some(0x0f), + vec![&mouse_irq], + )); + + // 16550A serial port (hw/char/serial-isa.c). + let com_ports = IO::new(0x3f8, 0x3f8, 0x00, 0x08); + let com_irq = if modern_serial_irq { + Irq::level_active_low_shared(4) + } else { + Irq::no_flags(4) + }; + out.extend(isa_device( + "COM1", + "PNP0501", + Some(1), + Some(0x0f), + vec![&com_ports, &com_irq], + )); + + // MC146818 RTC. QEMU only answers on the first two ports but reserves + // eight, following physical hardware (hw/rtc/mc146818rtc.c). + let rtc_ports = IO::new(0x70, 0x70, 0x01, 0x08); + let rtc_irq = Irq::no_flags(8); + out.extend(isa_device( + "RTC_", + "PNP0B00", + None, + None, + vec![&rtc_ports, &rtc_irq], + )); + + out +} + +/// `Device (name) { _HID, [_UID], [_STA], _CRS }`, the shape every ISA device +/// here shares. +fn isa_device( + name: &str, + hid: &str, + uid: Option<u8>, + sta: Option<u8>, + resources: Vec<&dyn Aml>, +) -> Vec<u8> { + let hid = Name::new(Path::new("_HID"), &EISAName::new(hid)); + let uid = uid.map(|uid| Name::new(Path::new("_UID"), &uid)); + let sta = sta.map(|sta| Name::new(Path::new("_STA"), &sta)); + let template = ResourceTemplate::new(resources); + let crs = Name::new(Path::new("_CRS"), &template); + + let mut children: Vec<&dyn Aml> = vec![&hid]; + children.extend(uid.iter().map(|name| name as &dyn Aml)); + children.extend(sta.iter().map(|name| name as &dyn Aml)); + children.push(&crs); + + emit(&Device::new(Path::new(name), children)) +} + +/// The short IRQ resource descriptor, ACPI 6.5 §6.4.2.1. `acpi_tables` only +/// models the extended form (`Interrupt`), which is not what QEMU emits for +/// these legacy devices. +struct Irq { + mask: u16, + /// `None` selects the two-byte `IRQNoFlags ()` form. + flags: Option<u8>, +} + +impl Irq { + /// `IRQNoFlags () {irq}`: edge triggered, active high, exclusive. + fn no_flags(irq: u8) -> Self { + Self { + mask: 1u16 << irq, + flags: None, + } + } + + /// `IRQ (Level, ActiveLow, Shared, ) {irq}`. + fn level_active_low_shared(irq: u8) -> Self { + // Bit 0 clear is level triggered, bit 3 is active low, bit 4 is shared. + Self { + mask: 1u16 << irq, + flags: Some((1 << 3) | (1 << 4)), + } + } +} + +impl Aml for Irq { + fn to_aml_bytes(&self, sink: &mut dyn AmlSink) { + match self.flags { + None => { + sink.byte(0x22); + sink.word(self.mask); + } + Some(flags) => { + sink.byte(0x23); + sink.word(self.mask); + sink.byte(flags); + } + } + } +} + +#[cfg(test)] +mod tests { + #[test] + fn matches_qemu() { + super::super::fixture::assert_region(&super::build(5, 0, true, None), 7834, 8245); + } +} diff --git a/dstack/crates/qemu-acpi/src/dsdt/ops.rs b/dstack/crates/qemu-acpi/src/dsdt/ops.rs new file mode 100644 index 000000000..68213c4b7 --- /dev/null +++ b/dstack/crates/qemu-acpi/src/dsdt/ops.rs @@ -0,0 +1,127 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! AML terms QEMU emits that `acpi_tables` does not model, plus the helper +//! that turns any term into bytes. +//! +//! Keep this module small: anything that exists upstream should be used from +//! upstream, so this stays a list of genuine gaps rather than a second builder. + +use acpi_tables::{Aml, AmlSink}; + +/// Serialize one AML term. +pub(crate) fn emit(term: &dyn Aml) -> Vec<u8> { + let mut bytes = Vec::new(); + term.to_aml_bytes(&mut bytes); + bytes +} + +/// Serialize a list of AML terms in order. +pub(crate) fn emit_all(terms: &[&dyn Aml]) -> Vec<u8> { + let mut bytes = Vec::new(); + for term in terms { + term.to_aml_bytes(&mut bytes); + } + bytes +} + +macro_rules! object_op { + ($(#[$doc:meta])* $name:ident, $opcode:expr) => { + $(#[$doc])* + pub(crate) struct $name<'a> { + operand: &'a dyn Aml, + } + + impl<'a> $name<'a> { + pub(crate) fn new(operand: &'a dyn Aml) -> Self { + Self { operand } + } + } + + impl Aml for $name<'_> { + fn to_aml_bytes(&self, sink: &mut dyn AmlSink) { + sink.byte($opcode); + self.operand.to_aml_bytes(sink); + } + } + }; +} + +object_op!( + /// `Increment (operand)` + Increment, + 0x75 +); +object_op!( + /// `LNot (operand)` + LNot, + 0x92 +); + +/// `ToHexString (operand, target)` +pub(crate) struct ToHexString<'a> { + operand: &'a dyn Aml, + target: &'a dyn Aml, +} + +impl<'a> ToHexString<'a> { + pub(crate) fn new(target: &'a dyn Aml, operand: &'a dyn Aml) -> Self { + Self { operand, target } + } +} + +impl Aml for ToHexString<'_> { + fn to_aml_bytes(&self, sink: &mut dyn AmlSink) { + sink.byte(0x98); + self.operand.to_aml_bytes(sink); + self.target.to_aml_bytes(sink); + } +} + +/// AML `PkgLength` for a term body of `content` bytes, using the same width +/// rule as QEMU: the smallest encoding that still fits once the length bytes +/// are counted in. +pub(crate) fn pkg_length(content: usize) -> Vec<u8> { + let width = if content < (1 << 6) - 1 { + 1 + } else if content < (1 << 12) - 2 { + 2 + } else if content < (1 << 20) - 3 { + 3 + } else { + 4 + }; + let length = content + width; + let mut out = Vec::with_capacity(width); + if width == 1 { + out.push(length as u8); + return out; + } + out.push((((width - 1) as u8) << 6) | (length & 0x0f) as u8); + for index in 0..width - 1 { + out.push((length >> (4 + index * 8)) as u8); + } + out +} + +/// `Scope (\) { .. }`. `acpi_tables::aml::Path` requires four-character name +/// segments, so it cannot express the root scope's null name. +pub(crate) fn root_scope(children: &[u8]) -> Vec<u8> { + let mut body = vec![b'\\', 0x00]; + body.extend_from_slice(children); + let mut out = vec![0x10]; + out.extend(pkg_length(body.len())); + out.extend(body); + out +} + +/// Raw pre-encoded AML, for terms that are more readable as bytes than as a +/// tree. Used sparingly and always next to the ASL it encodes. +pub(crate) struct Raw<'a>(pub(crate) &'a [u8]); + +impl Aml for Raw<'_> { + fn to_aml_bytes(&self, sink: &mut dyn AmlSink) { + sink.vec(self.0); + } +} diff --git a/dstack/crates/qemu-acpi/src/dsdt/pci0.rs b/dstack/crates/qemu-acpi/src/dsdt/pci0.rs new file mode 100644 index 000000000..3b6d17399 --- /dev/null +++ b/dstack/crates/qemu-acpi/src/dsdt/pci0.rs @@ -0,0 +1,251 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! `Scope (_SB)`: the PCI Express host bridge and the DRAM controller stub. +//! +//! ```asl +//! Scope (_SB) { +//! Device (PCI0) { +//! Name (_HID, EisaId ("PNP0A08")) +//! Name (_CID, EisaId ("PNP0A03")) +//! Name (_UID, Zero) +//! Method (_OSC, 4, NotSerialized) { +//! CreateDWordField (Arg3, Zero, CDW1) +//! If ((Arg0 == ToUUID ("33db4d5b-1ff7-401c-9657-7441c03dd766"))) { +//! CreateDWordField (Arg3, 0x04, CDW2) +//! CreateDWordField (Arg3, 0x08, CDW3) +//! Local0 = CDW3 +//! Local0 &= 0x1E +//! If ((Arg1 != One)) { +//! CDW1 |= 0x08 +//! } +//! If ((CDW3 != Local0)) { +//! CDW1 |= 0x10 +//! } +//! CDW3 = Local0 +//! } Else { +//! CDW1 |= 0x04 +//! } +//! Return (Arg3) +//! } +//! Method (EDSM, 5, Serialized) { +//! If ((Arg2 == Zero)) { +//! Local0 = Buffer (One) { 0x00 } +//! If ((Arg0 != ToUUID ("e5c937d0-3553-4d7a-9117-ea4d19c3434d"))) { +//! Return (Local0) +//! } +//! If ((Arg1 < 0x02)) { +//! Return (Local0) +//! } +//! Local0 [Zero] = 0x81 +//! Return (Local0) +//! } +//! If ((Arg2 == 0x07)) { +//! Local0 = Package (0x02) { Zero, "" } +//! Local1 = DerefOf (Arg4 [Zero]) +//! Local0 [Zero] = Local1 +//! Return (Local0) +//! } +//! } +//! } +//! Device (DRAC) { +//! Name (_HID, "PNP0C01") +//! Name (_CRS, ResourceTemplate () { +//! DWordMemory (ResourceProducer, PosDecode, MinFixed, MaxFixed, +//! NonCacheable, ReadWrite, +//! 0x00000000, // Granularity +//! 0xE0000000, // Range Minimum +//! 0xEFFFFFFF, // Range Maximum +//! 0x00000000, // Translation Offset +//! 0x10000000, // Length +//! ,, , AddressRangeMemory, TypeStatic) +//! }) +//! } +//! } +//! ``` + +use acpi_tables::aml::{ + AddressSpace, AddressSpaceCacheable, AmlStr, And, Arg, BufferData, CreateDWordField, DeRefOf, + Device, EISAName, Else, Equal, If, Index, LessThan, Local, Method, Name, NotEqual, One, Or, + Package, Path, ResourceTemplate, Return, Scope, Store, Uuid, Zero, +}; + +use super::ops::{emit, Raw}; + +/// PCI Firmware Specification host bridge `_OSC` UUID. +const OSC_UUID: &str = "33db4d5b-1ff7-401c-9657-7441c03dd766"; +/// PCI Firmware Specification device labeling `_DSM` UUID, the only function +/// set `EDSM` implements. +const DSM_LABEL_UUID: &str = "e5c937d0-3553-4d7a-9117-ea4d19c3434d"; + +/// The MMCONFIG window `DRAC` reserves: 256 MiB at 0xE0000000. +const MCFG_BASE: u32 = 0xE000_0000; +const MCFG_LAST: u32 = 0xEFFF_FFFF; + +pub(crate) fn build(pci_hotplug: bool) -> Vec<u8> { + let pci0 = pci0(pci_hotplug); + let drac = drac(); + emit(&Scope::new( + Path::new("_SB_"), + vec![&Raw(&pci0), &Raw(&drac)], + )) +} + +/// `Device (PCI0)`: the PCIe host bridge, its `_OSC` capability negotiation +/// and the `EDSM` helper every per-slot `_DSM` forwards to. +fn pci0(pci_hotplug: bool) -> Vec<u8> { + let hid = Name::new(Path::new("_HID"), &EISAName::new("PNP0A08")); + let cid = Name::new(Path::new("_CID"), &EISAName::new("PNP0A03")); + let uid = Name::new(Path::new("_UID"), &Zero {}); + let osc = osc(pci_hotplug); + let edsm = edsm(); + + emit(&Device::new( + Path::new("PCI0"), + vec![&hid, &cid, &uid, &Raw(&osc), &Raw(&edsm)], + )) +} + +/// `Method (_OSC, 4, NotSerialized)`. Arg3 is the capability buffer the OS +/// passes in; the method masks it down to what QEMU actually supports and +/// records the outcome in the status dword. +pub(crate) fn osc(pci_hotplug: bool) -> Vec<u8> { + let (arg0, arg1, arg3) = (Arg(0), Arg(1), Arg(3)); + let local0 = Local(0); + let zero = Zero {}; + let one = One {}; + let (cdw1, cdw2, cdw3) = (Path::new("CDW1"), Path::new("CDW2"), Path::new("CDW3")); + + let create_cdw1 = CreateDWordField::new(&cdw1, &arg3, &zero); + + let uuid = Uuid::new(OSC_UUID); + let recognized = Equal::new(&arg0, &uuid); + + let create_cdw2 = CreateDWordField::new(&cdw2, &arg3, &0x04u8); + let create_cdw3 = CreateDWordField::new(&cdw3, &arg3, &0x08u8); + let load = Store::new(&local0, &cdw3); + // keep only the control bits QEMU grants: SHPC, PME, AER, PCIe capability + let supported: u8 = if pci_hotplug { 0x1e } else { 0x1f }; + let mask = And::new(&local0, &local0, &supported); + + let wrong_revision = NotEqual::new(&arg1, &one); + let set_revision_error = Or::new(&cdw1, &cdw1, &0x08u8); + let revision_check = If::new(&wrong_revision, vec![&set_revision_error]); + + let masked_off = NotEqual::new(&cdw3, &local0); + let set_capability_error = Or::new(&cdw1, &cdw1, &0x10u8); + let capability_check = If::new(&masked_off, vec![&set_capability_error]); + + let store_back = Store::new(&cdw3, &local0); + let granted = If::new( + &recognized, + vec![ + &create_cdw2, + &create_cdw3, + &load, + &mask, + &revision_check, + &capability_check, + &store_back, + ], + ); + + // unrecognized UUID: set the "unrecognized" bit and grant nothing + let set_uuid_error = Or::new(&cdw1, &cdw1, &0x04u8); + let rejected = Else::new(vec![&set_uuid_error]); + + let ret = Return::new(&arg3); + + emit(&Method::new( + Path::new("_OSC"), + 4, + false, + vec![&create_cdw1, &granted, &rejected, &ret], + )) +} + +/// `Method (EDSM, 5, Serialized)`: the shared body of every PCI slot `_DSM`. +/// Arg0..Arg3 are the `_DSM` arguments; Arg4 is the slot's label package, +/// supplied by the caller so this body can be emitted once. +fn edsm() -> Vec<u8> { + let (arg0, arg1, arg2, arg4) = (Arg(0), Arg(1), Arg(2), Arg(4)); + let (local0, local1) = (Local(0), Local(1)); + let zero = Zero {}; + + let ret_local0 = Return::new(&local0); + + // function 0: report the supported function bitmap + let query = Equal::new(&arg2, &zero); + let empty = BufferData::new(vec![0x00]); + let init = Store::new(&local0, &empty); + + let label_uuid = Uuid::new(DSM_LABEL_UUID); + let other_uuid = NotEqual::new(&arg0, &label_uuid); + let uuid_check = If::new(&other_uuid, vec![&ret_local0]); + + let old_revision = LessThan::new(&arg1, &0x02u8); + let revision_check = If::new(&old_revision, vec![&ret_local0]); + + // functions 0 and 7 are supported, so bits 0 and 7 of the first byte + let slot0 = Index::new(&zero, &local0, &zero); + let set_bitmap = Store::new(&slot0, &0x81u8); + + let query_branch = If::new( + &query, + vec![ + &init, + &uuid_check, + &revision_check, + &set_bitmap, + &ret_local0, + ], + ); + + // function 7: return {slot number, label}, with the slot number lifted + // out of the caller-supplied package + let label = Equal::new(&arg2, &0x07u8); + let blank: AmlStr = ""; + let template = Package::new(vec![&zero, &blank]); + let alloc = Store::new(&local0, &template); + + let caller_slot0 = Index::new(&zero, &arg4, &zero); + let deref = DeRefOf::new(&caller_slot0); + let take_slot = Store::new(&local1, &deref); + let put_slot = Store::new(&slot0, &local1); + + let label_branch = If::new(&label, vec![&alloc, &take_slot, &put_slot, &ret_local0]); + + emit(&Method::new( + Path::new("EDSM"), + 5, + true, + vec![&query_branch, &label_branch], + )) +} + +/// `Device (DRAC)`: the DRAM controller. It exists only to claim the MMCONFIG +/// window as a producer, so the OS keeps it out of the PCI resource pool. +fn drac() -> Vec<u8> { + let hid_value: AmlStr = "PNP0C01"; + let hid = Name::new(Path::new("_HID"), &hid_value); + + let window = AddressSpace::new_memory( + AddressSpaceCacheable::NotCacheable, + true, + MCFG_BASE, + MCFG_LAST, + None, + ); + let crs = Name::new(Path::new("_CRS"), &ResourceTemplate::new(vec![&window])); + + emit(&Device::new(Path::new("DRAC"), vec![&hid, &crs])) +} + +#[cfg(test)] +mod tests { + #[test] + fn matches_qemu() { + super::super::fixture::assert_region(&super::build(true), 110, 426); + } +} diff --git a/dstack/crates/qemu-acpi/src/dsdt/pcihp.rs b/dstack/crates/qemu-acpi/src/dsdt/pcihp.rs new file mode 100644 index 000000000..ce7014436 --- /dev/null +++ b/dstack/crates/qemu-acpi/src/dsdt/pcihp.rs @@ -0,0 +1,340 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! `Scope (_SB.PCI0)`: the ACPI PCI hotplug registers and the methods that +//! drive them. +//! +//! ```asl +//! Scope (_SB.PCI0) { +//! OperationRegion (PCST, SystemIO, 0x0CC0, 0x08) +//! Field (PCST, DWordAcc, NoLock, WriteAsZeros) { PCIU, 32, PCID, 32 } +//! OperationRegion (SEJ, SystemIO, 0x0CC8, 0x04) +//! Field (SEJ, DWordAcc, NoLock, WriteAsZeros) { B0EJ, 32 } +//! OperationRegion (BNMR, SystemIO, 0x0CD0, 0x08) +//! Field (BNMR, DWordAcc, NoLock, WriteAsZeros) { BNUM, 32, PIDX, 32 } +//! Mutex (BLCK, 0x00) +//! Method (PCEJ, 2, NotSerialized) { +//! Acquire (BLCK, 0xFFFF) +//! BNUM = Arg0 +//! B0EJ = (One << Arg1) +//! Release (BLCK) +//! Return (Zero) +//! } +//! Method (AIDX, 2, NotSerialized) { +//! Acquire (BLCK, 0xFFFF) +//! BNUM = Arg0 +//! PIDX = (One << Arg1) +//! Local0 = PIDX +//! Release (BLCK) +//! Return (Local0) +//! } +//! Method (PDSM, 5, Serialized) { +//! If ((Arg2 == Zero)) { +//! Local0 = Buffer (One) { 0x00 } +//! If ((Arg0 != ToUUID ("e5c937d0-3553-4d7a-9117-ea4d19c3434d"))) { +//! Return (Local0) +//! } +//! If ((Arg1 < 0x02)) { +//! Return (Local0) +//! } +//! Local1 = Zero +//! Local2 = AIDX (DerefOf (Arg4 [Zero]), DerefOf (Arg4 [One])) +//! If (!((Local2 == Zero) | (Local2 == 0xFFFFFFFF))) { +//! Local1 |= One +//! Local1 |= (One << 0x07) +//! } +//! Local0 [Zero] = Local1 +//! Return (Local0) +//! } +//! If ((Arg2 == 0x07)) { +//! Local2 = AIDX (DerefOf (Arg4 [Zero]), DerefOf (Arg4 [One])) +//! Local0 = Package (0x02) {} +//! If (!((Local2 == Zero) || (Local2 == 0xFFFFFFFF))) { +//! Local0 [Zero] = Local2 +//! Local0 [One] = "" +//! } +//! Return (Local0) +//! } +//! } +//! } +//! ``` + +use acpi_tables::aml::{ + Acquire, AmlStr, Arg, BufferData, DeRefOf, Equal, Field, FieldAccessType, FieldEntry, + FieldLockRule, FieldUpdateRule, If, Index, LessThan, Local, Method, MethodCall, Mutex, + NotEqual, One, OpRegion, OpRegionSpace, Or, Path, Release, Return, Scope, ShiftLeft, Store, + Uuid, Zero, +}; +use acpi_tables::{Aml, AmlSink}; + +use super::ops::{emit, emit_all, pkg_length, LNot}; + +/// `LOr (a, b)`. `acpi_tables` models the comparison operators and the bitwise +/// ones, but not the logical connectives, and QEMU uses both `Or` and `LOr` in +/// `PDSM` on otherwise identical operands. +struct LOr<'a> { + a: &'a dyn Aml, + b: &'a dyn Aml, +} + +impl<'a> LOr<'a> { + fn new(a: &'a dyn Aml, b: &'a dyn Aml) -> Self { + Self { a, b } + } +} + +impl Aml for LOr<'_> { + fn to_aml_bytes(&self, sink: &mut dyn AmlSink) { + sink.byte(0x91); + self.a.to_aml_bytes(sink); + self.b.to_aml_bytes(sink); + } +} + +/// `Package (count) {}`: a package that reserves element slots but initializes +/// none of them. `Package` and `PackageBuilder` both derive the count from the +/// elements they are given, so neither can express the empty form. +struct EmptyPackage(u8); + +impl Aml for EmptyPackage { + fn to_aml_bytes(&self, sink: &mut dyn AmlSink) { + sink.byte(0x12); + sink.vec(&pkg_length(1)); + sink.byte(self.0); + } +} + +struct LegacyPair; + +impl Aml for LegacyPair { + fn to_aml_bytes(&self, sink: &mut dyn AmlSink) { + sink.byte(0x12); + sink.vec(&pkg_length(4)); + sink.byte(2); + sink.byte(0); // Zero + sink.byte(0x0d); // empty string + sink.byte(0); + } +} + +pub(crate) fn build(modern_device_label: bool) -> Vec<u8> { + let mut children = registers(); + children.extend(pcej()); + children.extend(aidx()); + children.extend(pdsm(modern_device_label)); + Scope::raw(Path::new("_SB_.PCI0"), children) +} + +/// The three hotplug I/O windows, their fields, and the mutex that serializes +/// the two-step "select a bus, then act on it" register protocol. +fn registers() -> Vec<u8> { + let dword_field = |region: &str, entries: Vec<FieldEntry>| { + Field::new( + Path::new(region), + FieldAccessType::DWord, + FieldLockRule::NoLock, + FieldUpdateRule::WriteAsZeroes, + entries, + ) + }; + + let pcst = OpRegion::new( + Path::new("PCST"), + OpRegionSpace::SystemIO, + &0x0cc0u16, + &0x08u8, + ); + let pcst_field = dword_field( + "PCST", + vec![ + FieldEntry::Named(*b"PCIU", 32), + FieldEntry::Named(*b"PCID", 32), + ], + ); + + let sej = OpRegion::new( + Path::new("SEJ_"), + OpRegionSpace::SystemIO, + &0x0cc8u16, + &0x04u8, + ); + let sej_field = dword_field("SEJ_", vec![FieldEntry::Named(*b"B0EJ", 32)]); + + let bnmr = OpRegion::new( + Path::new("BNMR"), + OpRegionSpace::SystemIO, + &0x0cd0u16, + &0x08u8, + ); + let bnmr_field = dword_field( + "BNMR", + vec![ + FieldEntry::Named(*b"BNUM", 32), + FieldEntry::Named(*b"PIDX", 32), + ], + ); + + let lock = Mutex::new(Path::new("BLCK"), 0x00); + + emit_all(&[ + &pcst, + &pcst_field, + &sej, + &sej_field, + &bnmr, + &bnmr_field, + &lock, + ]) +} + +/// `PCEJ (bus, slot)`: eject the slot by writing its bit to `B0EJ`. +fn pcej() -> Vec<u8> { + let (bnum, b0ej) = (Path::new("BNUM"), Path::new("B0EJ")); + let (arg0, arg1) = (Arg(0), Arg(1)); + + let acquire = Acquire::new(Path::new("BLCK"), 0xffff); + let select = Store::new(&bnum, &arg0); + let slot_bit = ShiftLeft::new(&Zero {}, &One {}, &arg1); + let eject = Store::new(&b0ej, &slot_bit); + let release = Release::new(Path::new("BLCK")); + let ret = Return::new(&Zero {}); + + emit(&Method::new( + Path::new("PCEJ"), + 2, + false, + vec![&acquire, &select, &eject, &release, &ret], + )) +} + +/// `AIDX (bus, slot)`: read back the ACPI index the firmware assigned to the +/// slot, or 0/0xFFFFFFFF when the host does not report one. +fn aidx() -> Vec<u8> { + let local0 = Local(0); + let (bnum, pidx) = (Path::new("BNUM"), Path::new("PIDX")); + let (arg0, arg1) = (Arg(0), Arg(1)); + + let acquire = Acquire::new(Path::new("BLCK"), 0xffff); + let select = Store::new(&bnum, &arg0); + let slot_bit = ShiftLeft::new(&Zero {}, &One {}, &arg1); + let request = Store::new(&pidx, &slot_bit); + let read = Store::new(&local0, &pidx); + let release = Release::new(Path::new("BLCK")); + let ret = Return::new(&local0); + + emit(&Method::new( + Path::new("AIDX"), + 2, + false, + vec![&acquire, &select, &request, &read, &release, &ret], + )) +} + +/// `PDSM (uuid, revision, function, args, slot)`: the `_DSM` body shared by +/// every hotplug-capable slot. Function 0 reports which functions exist, +/// function 7 returns the slot's ACPI index and (empty) label. +fn pdsm(modern_device_label: bool) -> Vec<u8> { + let (local0, local1, local2) = (Local(0), Local(1), Local(2)); + let (arg0, arg1, arg2, arg4) = (Arg(0), Arg(1), Arg(2), Arg(4)); + + // shared by both branches: AIDX (DerefOf (Arg4 [Zero]), DerefOf (Arg4 [One])) + let bus_index = Index::new(&Zero {}, &arg4, &Zero {}); + let bus = DeRefOf::new(&bus_index); + let slot_index = Index::new(&Zero {}, &arg4, &One {}); + let slot = DeRefOf::new(&slot_index); + let aidx_call = MethodCall::new(Path::new("AIDX"), vec![&bus, &slot]); + let read_index = Store::new(&local2, &aidx_call); + + // shared by both branches: the "no index reported" test, once as a bitwise + // Or and once as a logical LOr, matching QEMU term for term. + let unassigned = Equal::new(&local2, &Zero {}); + let invalid = Equal::new(&local2, &0xffff_ffffu32); + let no_index_bitwise = Or::new(&Zero {}, &unassigned, &invalid); + let has_index_bitwise = LNot::new(&no_index_bitwise); + let no_index_logical = LOr::new(&unassigned, &invalid); + let has_index_logical = LNot::new(&no_index_logical); + + let ret0 = Return::new(&local0); + + // If ((Arg2 == Zero)): the supported-function bitmap. + let empty_bitmap = BufferData::new(vec![0x00]); + let init_bitmap = Store::new(&local0, &empty_bitmap); + + let uuid = Uuid::new("e5c937d0-3553-4d7a-9117-ea4d19c3434d"); + let wrong_uuid = NotEqual::new(&arg0, &uuid); + let bail_uuid = If::new(&wrong_uuid, vec![&ret0]); + let old_revision = LessThan::new(&arg1, &0x02u8); + let bail_revision = If::new(&old_revision, vec![&ret0]); + + let init_bits = Store::new(&local1, &Zero {}); + let function_zero = Or::new(&local1, &local1, &One {}); + let function_seven_bit = ShiftLeft::new(&Zero {}, &One {}, &0x07u8); + let function_seven = Or::new(&local1, &local1, &function_seven_bit); + let set_bits = If::new(&has_index_bitwise, vec![&function_zero, &function_seven]); + + let bitmap_slot = Index::new(&Zero {}, &local0, &Zero {}); + let store_bits = Store::new(&bitmap_slot, &local1); + let is_query = Equal::new(&arg2, &Zero {}); + let query = If::new( + &is_query, + vec![ + &init_bitmap, + &bail_uuid, + &bail_revision, + &init_bits, + &read_index, + &set_bits, + &store_bits, + &ret0, + ], + ); + + // If ((Arg2 == 0x07)): the device name, as (ACPI index, label) pair. + let empty_pair = EmptyPackage(0x02); + let legacy_pair = LegacyPair; + let init_empty_pair = Store::new(&local0, &empty_pair); + let init_legacy_pair = Store::new(&local0, &legacy_pair); + let init_pair_bytes = if modern_device_label { + emit(&init_empty_pair) + } else { + emit(&init_legacy_pair) + }; + let init_pair = super::ops::Raw(&init_pair_bytes); + let index_slot = Index::new(&Zero {}, &local0, &Zero {}); + let store_index = Store::new(&index_slot, &local2); + let label: AmlStr = ""; + let label_slot = Index::new(&Zero {}, &local0, &One {}); + let store_label = Store::new(&label_slot, &label); + let modern_fill = If::new(&has_index_logical, vec![&store_index, &store_label]); + let fill_pair_bytes = if modern_device_label { + emit(&modern_fill) + } else { + emit(&store_index) + }; + let fill_pair = super::ops::Raw(&fill_pair_bytes); + let is_name = Equal::new(&arg2, &0x07u8); + let modern_name = If::new(&is_name, vec![&read_index, &init_pair, &fill_pair, &ret0]); + let legacy_name = If::new(&is_name, vec![&init_pair, &read_index, &fill_pair, &ret0]); + let name_bytes = if modern_device_label { + emit(&modern_name) + } else { + emit(&legacy_name) + }; + let name = super::ops::Raw(&name_bytes); + + emit(&Method::new( + Path::new("PDSM"), + 5, + true, + vec![&query, &name], + )) +} + +#[cfg(test)] +mod tests { + #[test] + fn matches_qemu() { + super::super::fixture::assert_region(&super::build(true), 426, 777); + } +} diff --git a/dstack/crates/qemu-acpi/src/dsdt/pic.rs b/dstack/crates/qemu-acpi/src/dsdt/pic.rs new file mode 100644 index 000000000..26055a46f --- /dev/null +++ b/dstack/crates/qemu-acpi/src/dsdt/pic.rs @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! The interrupt model the OS selected. `_PIC` only records the choice; the +//! `_PRT` methods later in the table read `PICF` back to decide which routing +//! package to return. +//! +//! ```asl +//! Name (PICF, Zero) +//! Method (_PIC, 1, NotSerialized) { +//! PICF = Arg0 +//! } +//! ``` + +use acpi_tables::aml::{Arg, Method, Name, Path, Store, Zero}; + +use super::ops::emit_all; + +pub(crate) fn build() -> Vec<u8> { + let picf = Path::new("PICF"); + let flag = Name::new(Path::new("PICF"), &Zero {}); + let record = Store::new(&picf, &Arg(0)); + let method = Method::new(Path::new("_PIC"), 1, false, vec![&record]); + + emit_all(&[&flag, &method]) +} + +#[cfg(test)] +mod tests { + #[test] + fn matches_qemu() { + super::super::fixture::assert_region(&super::build(), 777, 796); + } +} diff --git a/dstack/crates/qemu-acpi/src/dsdt/prt.rs b/dstack/crates/qemu-acpi/src/dsdt/prt.rs new file mode 100644 index 000000000..67ce13b23 --- /dev/null +++ b/dstack/crates/qemu-acpi/src/dsdt/prt.rs @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! `Scope (PCI0)`: the two PCI interrupt routing tables and `_PRT`. +//! +//! `PRTP` (PIC mode) routes every slot/pin pair to a `LNKx` link device, +//! `PRTA` (APIC mode) routes it to the matching `GSIx` device. Both are a +//! `Package` of 128 entries — 32 slots by 4 pins — with identical shape: +//! +//! ```asl +//! Scope (PCI0) { +//! Name (PRTP, Package (0x80) { +//! Package (0x04) { 0xFFFF, Zero, LNKE, Zero }, // slot 0, INTA +//! ... // 127 more +//! }) +//! Name (PRTA, Package (0x80) { ... }) // same, GSIx +//! Method (_PRT, 0, NotSerialized) { +//! If ((PICF == Zero)) { Return (PRTP) } +//! Else { Return (PRTA) } +//! } +//! } +//! ``` +//! +//! Construction rule (QEMU `build_q35_routing_table()` / +//! `append_q35_prt_entry()` in `hw/i386/acpi-build.c`): entry `(slot, pin)` +//! holds the address `(slot << 16) | 0xffff`, the pin index, the link name, +//! and `Zero` for the (unused) source index. The link letter is +//! `base + (head + pin) % 4`, where the per-slot `base`/`head` follow the +//! chipset's default `D<N>IR` values: +//! +//! * slots `0x00..=0x17` — base `E`, head `slot & 3` (PIRQ\[E-H\], rotating) +//! * slot `0x18` — base `E`, head 0 +//! * slots `0x19..=0x1d` — base `A`, head 0 (INTA -> PIRQA) +//! * slot `0x1e` — base `E`, head 0 (PCIe->PCI bridge, PIRQ\[E-H\]) +//! * slot `0x1f` — base `A`, head 0 + +use acpi_tables::aml::{Else, Equal, If, Method, Name, PackageBuilder, Path, Return, Scope, Zero}; + +use super::ops::emit_all; + +/// One 128-entry routing table. `prefix` is the three-character device family +/// (`LNK` for PIC mode, `GSI` for APIC mode). +fn routing_table(prefix: &str) -> PackageBuilder { + let mut table = PackageBuilder::new(); + for slot in 0u32..32 { + let base = if matches!(slot, 0x19..=0x1d | 0x1f) { + b'A' + } else { + b'E' + }; + let head = if slot < 0x18 { slot as u8 & 3 } else { 0 }; + // Slot 0's address is 0xffff, which QEMU emits as a word rather than a + // dword; the `u32` encoder narrows to the same shortest form. + let address = (slot << 16) | 0xffff; + for pin in 0u8..4 { + let name = format!("{prefix}{}", (base + (head + pin) % 4) as char); + let mut entry = PackageBuilder::new(); + entry.add_element(&address); + entry.add_element(&pin); + entry.add_element(&Path::new(&name)); + entry.add_element(&Zero {}); + table.add_element(&entry); + } + } + table +} + +pub(crate) fn build() -> Vec<u8> { + let prtp = Name::new(Path::new("PRTP"), &routing_table("LNK")); + let prta = Name::new(Path::new("PRTA"), &routing_table("GSI")); + + let picf = Path::new("PICF"); + let zero = Zero {}; + let pic_mode = Equal::new(&picf, &zero); + let pic_table = Path::new("PRTP"); + let apic_table = Path::new("PRTA"); + let return_pic = Return::new(&pic_table); + let return_apic = Return::new(&apic_table); + let if_pic = If::new(&pic_mode, vec![&return_pic]); + let else_apic = Else::new(vec![&return_apic]); + let prt = Method::new(Path::new("_PRT"), 0, false, vec![&if_pic, &else_apic]); + + Scope::raw(Path::new("PCI0"), emit_all(&[&prtp, &prta, &prt])) +} + +#[cfg(test)] +mod tests { + #[test] + fn matches_qemu() { + super::super::fixture::assert_region(&super::build(), 804, 4552); + } +} diff --git a/dstack/crates/qemu-acpi/src/dsdt/pxb.rs b/dstack/crates/qemu-acpi/src/dsdt/pxb.rs new file mode 100644 index 000000000..439e5369f --- /dev/null +++ b/dstack/crates/qemu-acpi/src/dsdt/pxb.rs @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// SPDX-License-Identifier: Apache-2.0 + +//! ACPI host bridge emitted for dstack's GPU PXB. + +use acpi_tables::aml::{ + AddressSpace, Device, EISAName, Method, Name, PackageBuilder, Path, ResourceTemplate, Return, + Scope, Zero, +}; + +use super::ops::{emit, Raw}; + +const BUS: u8 = 5; + +fn routing_table() -> PackageBuilder { + let mut table = PackageBuilder::new(); + for slot in 0u32..32 { + let address = (slot << 16) | 0xffff; + for pin in 0u8..4 { + let letter = (b'A' + ((slot as u8 + pin + 3) & 3)) as char; + let link = Path::new(&format!("LNK{letter}")); + let mut entry = PackageBuilder::new(); + entry.add_element(&address); + entry.add_element(&pin); + entry.add_element(&link); + entry.add_element(&Zero {}); + table.add_element(&entry); + } + } + table +} + +pub(crate) fn build() -> Vec<u8> { + let uid = Name::new(Path::new("_UID"), &BUS); + let bbn = Name::new(Path::new("_BBN"), &BUS); + let hid = Name::new(Path::new("_HID"), &EISAName::new("PNP0A08")); + let cid = Name::new(Path::new("_CID"), &EISAName::new("PNP0A03")); + let osc = super::pci0::osc(false); + let pxm = Name::new(Path::new("_PXM"), &Zero {}); + + let routes = routing_table(); + let return_routes = Return::new(&routes); + let prt = Method::new(Path::new("_PRT"), 0, false, vec![&return_routes]); + + let buses = AddressSpace::new_bus_number(u16::from(BUS), u16::from(BUS)); + let resources = ResourceTemplate::new(vec![&buses]); + let crs = Name::new(Path::new("_CRS"), &resources); + + let osc = Raw(&osc); + let bridge = Device::new( + Path::new("PC05"), + vec![&uid, &bbn, &hid, &cid, &osc, &pxm, &prt, &crs], + ); + Scope::raw(Path::new("\\_SB_"), emit(&bridge)) +} diff --git a/dstack/crates/qemu-acpi/src/dsdt/sstate.rs b/dstack/crates/qemu-acpi/src/dsdt/sstate.rs new file mode 100644 index 000000000..25a39b85b --- /dev/null +++ b/dstack/crates/qemu-acpi/src/dsdt/sstate.rs @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! `Scope (\)`: the sleep state packages QEMU advertises for S3/S4/S5. +//! +//! ```asl +//! Scope (\) { +//! Name (_S3, Package (0x04) { One, One, Zero, Zero }) +//! Name (_S4, Package (0x04) { 0x02, 0x02, Zero, Zero }) +//! Name (_S5, Package (0x04) { Zero, Zero, Zero, Zero }) +//! } +//! ``` + +use acpi_tables::aml::{Name, One, Package, Path, Zero}; + +use super::ops::{emit_all, root_scope}; + +pub(crate) fn build() -> Vec<u8> { + let one = One {}; + let zero = Zero {}; + let two = 2u8; + + // Each package is { PM1a SLP_TYP, PM1b SLP_TYP, reserved, reserved }. + let s3 = Package::new(vec![&one, &one, &zero, &zero]); + let s4 = Package::new(vec![&two, &two, &zero, &zero]); + let s5 = Package::new(vec![&zero, &zero, &zero, &zero]); + + let s3 = Name::new(Path::new("_S3_"), &s3); + let s4 = Name::new(Path::new("_S4_"), &s4); + let s5 = Name::new(Path::new("_S5_"), &s5); + + root_scope(&emit_all(&[&s3, &s4, &s5])) +} + +#[cfg(test)] +mod tests { + #[test] + fn matches_qemu() { + super::super::fixture::assert_region(&super::build(), 7732, 7774); + } +} diff --git a/dstack/crates/qemu-acpi/src/fixed_tables.rs b/dstack/crates/qemu-acpi/src/fixed_tables.rs new file mode 100644 index 000000000..1462aed6e --- /dev/null +++ b/dstack/crates/qemu-acpi/src/fixed_tables.rs @@ -0,0 +1,207 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Fixed-layout ACPI tables emitted by QEMU's Q35 machine. +//! +//! Checksums stay zero in these blobs. QEMU's table-loader asks firmware to +//! calculate them after it has relocated the table pointers. + +const OEM_ID: &[u8; 6] = b"BOCHS "; +const OEM_TABLE_ID: &[u8; 8] = b"BXPC "; +const CREATOR_ID: &[u8; 4] = b"BXPC"; + +const DSDT_OFFSET: u32 = 64; +const PM_IO_BASE: u32 = 0x0600; +const GPE0_IO_BASE: u32 = 0x0620; +const SCI_INTERRUPT: u16 = 9; +const SMI_COMMAND_PORT: u32 = 0x00b2; +const ACPI_ENABLE_COMMAND: u8 = 2; +const ACPI_DISABLE_COMMAND: u8 = 3; +const LOCAL_APIC_ADDRESS: u32 = 0xfee0_0000; +const IO_APIC_ADDRESS: u32 = 0xfec0_0000; +const MCFG_BASE: u64 = 0xe000_0000; + +fn put(out: &mut [u8], offset: usize, bytes: &[u8]) { + out[offset..offset + bytes.len()].copy_from_slice(bytes); +} + +/// Standard 36-byte ACPI system-description-table header. +fn header(signature: &[u8; 4], length: usize, revision: u8) -> Vec<u8> { + let mut out = vec![0; length]; + put(&mut out, 0, signature); + put(&mut out, 4, &(length as u32).to_le_bytes()); + out[8] = revision; + put(&mut out, 10, OEM_ID); + put(&mut out, 16, OEM_TABLE_ID); + put(&mut out, 24, &1u32.to_le_bytes()); + put(&mut out, 28, CREATOR_ID); + put(&mut out, 32, &1u32.to_le_bytes()); + out +} + +/// Firmware ACPI Control Structure. Unlike the other records it has no SDT +/// header and is never checksummed. +pub(crate) fn facs() -> Vec<u8> { + let mut out = vec![0; 64]; + put(&mut out, 0, b"FACS"); + put(&mut out, 4, &64u32.to_le_bytes()); + out +} + +/// Fixed ACPI Description Table for Q35/ICH9. +pub(crate) fn fadt(smm: bool, cpu_count: u32) -> Vec<u8> { + let mut out = header(b"FACP", 244, 3); + // Firmware patches FIRMWARE_CTRL; DSDT is blob-relative until relocation. + put(&mut out, 40, &DSDT_OFFSET.to_le_bytes()); + out[44] = 1; // Multiple APIC interrupt model. + put(&mut out, 46, &SCI_INTERRUPT.to_le_bytes()); + if smm { + put(&mut out, 48, &SMI_COMMAND_PORT.to_le_bytes()); + out[52] = ACPI_ENABLE_COMMAND; + out[53] = ACPI_DISABLE_COMMAND; + } + // Legacy fixed-register addresses and lengths. + put(&mut out, 56, &PM_IO_BASE.to_le_bytes()); // PM1a event block + put(&mut out, 64, &(PM_IO_BASE + 4).to_le_bytes()); // PM1a control block + put(&mut out, 76, &(PM_IO_BASE + 8).to_le_bytes()); // PM timer + put(&mut out, 80, &GPE0_IO_BASE.to_le_bytes()); + out[88] = 4; // PM1 event length + out[89] = 2; // PM1 control length + out[91] = 4; // PM timer length + out[92] = 16; // GPE0 block length + put(&mut out, 96, &0x0fffu16.to_le_bytes()); // C2 unsupported + put(&mut out, 98, &0x0fffu16.to_le_bytes()); // C3 unsupported + out[108] = 0x32; // RTC century register + put(&mut out, 109, &2u16.to_le_bytes()); // 8042 present + let mut flags = (1 << 0) | (1 << 2) | (1 << 5) | (1 << 7) | (1 << 10) | (1 << 15); + if cpu_count > 8 { + flags |= 1 << 18; // Force APIC clustered logical destination mode. + } + put(&mut out, 112, &(flags as u32).to_le_bytes()); + // Reset register: System I/O, byte access, port 0xcf9; reset value 0x0f. + out[116] = 1; + out[117] = 8; + put(&mut out, 120, &0x0cf9u64.to_le_bytes()); + out[128] = 0x0f; + // X_FIRMWARE_CTRL is zero; X_DSDT repeats the blob-relative DSDT offset. + put(&mut out, 140, &u64::from(DSDT_OFFSET).to_le_bytes()); + gas(&mut out, 148, 32, PM_IO_BASE); + gas(&mut out, 172, 16, PM_IO_BASE + 4); + gas(&mut out, 208, 32, PM_IO_BASE + 8); + gas(&mut out, 220, 128, GPE0_IO_BASE); + out +} + +fn gas(out: &mut [u8], offset: usize, width: u8, address: u32) { + out[offset] = 1; // System I/O + out[offset + 1] = width; + put(out, offset + 4, &u64::from(address).to_le_bytes()); +} + +/// Multiple APIC Description Table. +pub(crate) fn madt(cpu_count: u32, pic: bool, legacy_irq_overrides: bool) -> Vec<u8> { + let legacy_cpus = cpu_count.min(255) as usize; + let x2apic_cpus = cpu_count.saturating_sub(255) as usize; + let cpu_bytes = legacy_cpus * 8 + x2apic_cpus * 16; + let interrupt_tail = if legacy_irq_overrides { 172 } else { 62 }; + let lint_len = if cpu_count <= 255 { 6 } else { 12 }; + let tail_len = interrupt_tail + lint_len; + let mut out = header(b"APIC", 44 + cpu_bytes + tail_len, 3); + put(&mut out, 36, &LOCAL_APIC_ADDRESS.to_le_bytes()); + put(&mut out, 40, &(pic as u32).to_le_bytes()); + for index in 0..cpu_count { + if index < 255 { + let at = 44 + index as usize * 8; + put(&mut out, at, &[0, 8, index as u8, index as u8]); + put(&mut out, at + 4, &1u32.to_le_bytes()); + } else { + let at = 44 + legacy_cpus * 8 + (index as usize - 255) * 16; + put(&mut out, at, &[9, 16, 0, 0]); + put(&mut out, at + 4, &index.to_le_bytes()); + put(&mut out, at + 8, &1u32.to_le_bytes()); + put(&mut out, at + 12, &index.to_le_bytes()); + } + } + let at = 44 + cpu_bytes; + put(&mut out, at, &[1, 12, 0, 0]); // I/O APIC ID 0 + put(&mut out, at + 4, &IO_APIC_ADDRESS.to_le_bytes()); + if legacy_irq_overrides { + for irq in 0u8..16 { + let entry = at + 12 + usize::from(irq) * 10; + put(&mut out, entry, &[2, 10, 0, irq]); + let gsi = if irq == 0 { 2 } else { u32::from(irq) }; + put(&mut out, entry + 4, &gsi.to_le_bytes()); + put(&mut out, entry + 8, &5u16.to_le_bytes()); + } + } else { + // IRQ0 -> GSI2, then overrides for IRQ5/9/10/11 (level, active-low). + put(&mut out, at + 12, &[2, 10, 0, 0, 2, 0, 0, 0, 0, 0]); + for (n, irq) in [5u8, 9, 10, 11].into_iter().enumerate() { + let entry = at + 22 + n * 10; + put(&mut out, entry, &[2, 10, 0, irq]); + put(&mut out, entry + 4, &u32::from(irq).to_le_bytes()); + put(&mut out, entry + 8, &13u16.to_le_bytes()); + } + } + let lint = at + interrupt_tail; + if cpu_count <= 255 { + put(&mut out, lint, &[4, 6, 0xff, 0, 0, 1]); + } else { + put(&mut out, lint, &[0x0a, 12, 0, 0]); + put(&mut out, lint + 4, &u32::MAX.to_le_bytes()); + put(&mut out, lint + 8, &[1, 0, 0, 0]); + } + out +} + +pub(crate) fn mcfg() -> Vec<u8> { + let mut out = header(b"MCFG", 60, 1); + put(&mut out, 44, &MCFG_BASE.to_le_bytes()); + out[55] = 0xff; // buses 0..255, segment group zero + out +} + +pub(crate) fn waet() -> Vec<u8> { + let mut out = header(b"WAET", 40, 1); + put(&mut out, 36, &2u32.to_le_bytes()); // ACPI PM timer is reliable + out +} + +pub(crate) fn rsdt(entries: &[u32]) -> Vec<u8> { + let mut out = header(b"RSDT", 36 + entries.len() * 4, 1); + for (index, entry) in entries.iter().enumerate() { + put(&mut out, 36 + index * 4, &entry.to_le_bytes()); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn facs_matches_qemu() { + crate::dsdt::fixture::assert_blob_range(&facs(), 0, 64); + } + #[test] + fn fadt_matches_qemu() { + crate::dsdt::fixture::assert_blob_range(&fadt(false, 1), 8322, 8566); + } + #[test] + fn madt_matches_qemu() { + crate::dsdt::fixture::assert_blob_range(&madt(1, false, false), 8566, 8686); + } + #[test] + fn mcfg_matches_qemu() { + crate::dsdt::fixture::assert_blob_range(&mcfg(), 8686, 8746); + } + #[test] + fn waet_matches_qemu() { + crate::dsdt::fixture::assert_blob_range(&waet(), 8746, 8786); + } + #[test] + fn rsdt_matches_qemu() { + crate::dsdt::fixture::assert_blob_range(&rsdt(&[8322, 8566, 8686, 8746]), 8786, 8838); + } +} diff --git a/dstack/crates/qemu-acpi/src/fw_cfg.rs b/dstack/crates/qemu-acpi/src/fw_cfg.rs new file mode 100644 index 000000000..7fc976a3e --- /dev/null +++ b/dstack/crates/qemu-acpi/src/fw_cfg.rs @@ -0,0 +1,252 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// SPDX-License-Identifier: Apache-2.0 + +//! Construction of QEMU's `etc/table-loader` and `etc/acpi/rsdp` blobs. +//! +//! QEMU records one fixed-size (128-byte) loader command whenever an ACPI +//! table needs allocation, pointer relocation, or a checksum. Command order is +//! observable and therefore part of the measured ACPI ABI. + +use crate::{AcpiBlobs, Error}; + +const LOADER_COMMAND_SIZE: usize = 128; +const LOADER_FILE_NAME_SIZE: usize = 56; +const LOADER_BLOB_SIZE: usize = 4096; +const ACPI_HEADER_SIZE: u32 = 36; +const ACPI_CHECKSUM_OFFSET: u32 = 9; + +const TABLES_FILE: &str = "etc/acpi/tables"; +const RSDP_FILE: &str = "etc/acpi/rsdp"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct TableLocation { + offset: u32, + checksum_offset: u32, + length: u32, +} + +fn malformed(message: impl Into<String>) -> Error { + Error::MalformedTables(message.into()) +} + +fn append_file_name(output: &mut Vec<u8>, file_name: &str) -> Result<(), Error> { + if file_name.len() > LOADER_FILE_NAME_SIZE { + return Err(malformed("fw_cfg file name exceeds QEMU loader limit")); + } + output.extend_from_slice(file_name.as_bytes()); + output.resize(output.len() + LOADER_FILE_NAME_SIZE - file_name.len(), 0); + Ok(()) +} + +fn finish_command(output: &mut Vec<u8>, payload_size: usize) -> Result<(), Error> { + let padding = LOADER_COMMAND_SIZE + .checked_sub(payload_size) + .ok_or_else(|| malformed("fw_cfg loader command exceeds 128 bytes"))?; + output.resize(output.len() + padding, 0); + Ok(()) +} + +fn append_allocate( + output: &mut Vec<u8>, + file_name: &str, + alignment: u32, + zone: u8, +) -> Result<(), Error> { + output.extend_from_slice(&1u32.to_le_bytes()); + append_file_name(output, file_name)?; + output.extend_from_slice(&alignment.to_le_bytes()); + output.push(zone); + finish_command(output, 4 + LOADER_FILE_NAME_SIZE + 4 + 1) +} + +fn append_add_pointer( + output: &mut Vec<u8>, + destination_file: &str, + source_file: &str, + offset: u32, + pointer_size: u8, +) -> Result<(), Error> { + output.extend_from_slice(&2u32.to_le_bytes()); + append_file_name(output, destination_file)?; + append_file_name(output, source_file)?; + output.extend_from_slice(&offset.to_le_bytes()); + output.push(pointer_size); + finish_command(output, 4 + 2 * LOADER_FILE_NAME_SIZE + 4 + 1) +} + +fn append_add_checksum( + output: &mut Vec<u8>, + file_name: &str, + checksum_offset: u32, + start: u32, + length: u32, +) -> Result<(), Error> { + output.extend_from_slice(&3u32.to_le_bytes()); + append_file_name(output, file_name)?; + output.extend_from_slice(&checksum_offset.to_le_bytes()); + output.extend_from_slice(&start.to_le_bytes()); + output.extend_from_slice(&length.to_le_bytes()); + finish_command(output, 4 + LOADER_FILE_NAME_SIZE + 3 * 4) +} + +fn parse_table_locations(tables: &[u8]) -> Result<Vec<([u8; 4], TableLocation)>, Error> { + let mut result = Vec::new(); + let mut offset = 0usize; + + while offset < tables.len() { + // QEMU pads the blob with zeroes after the final table. + if tables.get(offset..offset + 4) == Some(&[0, 0, 0, 0]) { + break; + } + let signature: [u8; 4] = tables + .get(offset..offset.saturating_add(4)) + .and_then(|bytes| bytes.try_into().ok()) + .ok_or_else(|| malformed("truncated ACPI table signature"))?; + let length = tables + .get(offset.saturating_add(4)..offset.saturating_add(8)) + .and_then(|bytes| <&[u8; 4]>::try_from(bytes).ok()) + .map(|bytes| u32::from_le_bytes(*bytes)) + .ok_or_else(|| malformed("truncated ACPI table header"))?; + if length < ACPI_HEADER_SIZE { + return Err(malformed("invalid ACPI table length")); + } + let next = offset + .checked_add(length as usize) + .filter(|next| *next <= tables.len()) + .ok_or_else(|| malformed("ACPI table extends beyond blob"))?; + let table_offset = u32::try_from(offset) + .map_err(|_| malformed("ACPI table offset exceeds 32-bit loader ABI"))?; + let checksum_offset = table_offset + .checked_add(ACPI_CHECKSUM_OFFSET) + .ok_or_else(|| malformed("ACPI checksum offset overflow"))?; + result.push(( + signature, + TableLocation { + offset: table_offset, + checksum_offset, + length, + }, + )); + offset = next; + } + Ok(result) +} + +fn required_table( + locations: &[([u8; 4], TableLocation)], + signature: &[u8; 4], +) -> Result<TableLocation, Error> { + locations + .iter() + .find_map(|(candidate, location)| (candidate == signature).then_some(*location)) + .ok_or_else(|| malformed(String::from_utf8_lossy(signature).into_owned())) +} + +fn optional_table( + locations: &[([u8; 4], TableLocation)], + signature: &[u8; 4], +) -> Option<TableLocation> { + locations + .iter() + .find_map(|(candidate, location)| (candidate == signature).then_some(*location)) +} + +fn append_table_checksum(output: &mut Vec<u8>, table: TableLocation) -> Result<(), Error> { + append_add_checksum( + output, + TABLES_FILE, + table.checksum_offset, + table.offset, + table.length, + ) +} + +pub(crate) fn finish(tables: Vec<u8>) -> Result<AcpiBlobs, Error> { + let locations = parse_table_locations(&tables)?; + let dsdt = required_table(&locations, b"DSDT")?; + let fadt = required_table(&locations, b"FACP")?; + let madt = required_table(&locations, b"APIC")?; + let srat = optional_table(&locations, b"SRAT"); + let mcfg = required_table(&locations, b"MCFG")?; + let waet = required_table(&locations, b"WAET")?; + let rsdt = required_table(&locations, b"RSDT")?; + + let mut rsdp = b"RSD PTR \0BOCHS \0".to_vec(); + rsdp.extend_from_slice(&rsdt.offset.to_le_bytes()); + + let mut loader = Vec::with_capacity(LOADER_BLOB_SIZE); + append_allocate(&mut loader, RSDP_FILE, 16, 2)?; + append_allocate(&mut loader, TABLES_FILE, 64, 1)?; + + // This sequence mirrors QEMU's build order. In particular, SRAT is built + // between MADT and MCFG only for NUMA configurations. + append_table_checksum(&mut loader, dsdt)?; + for (offset, pointer_size) in [(36, 4), (40, 4), (140, 8)] { + append_add_pointer( + &mut loader, + TABLES_FILE, + TABLES_FILE, + fadt.offset + .checked_add(offset) + .ok_or_else(|| malformed("FADT pointer offset overflow"))?, + pointer_size, + )?; + } + append_table_checksum(&mut loader, fadt)?; + append_table_checksum(&mut loader, madt)?; + if let Some(srat) = srat { + append_table_checksum(&mut loader, srat)?; + } + append_table_checksum(&mut loader, mcfg)?; + append_table_checksum(&mut loader, waet)?; + + let rsdt_payload_length = rsdt + .length + .checked_sub(ACPI_HEADER_SIZE) + .ok_or_else(|| malformed("RSDT shorter than ACPI header"))?; + if rsdt_payload_length % 4 != 0 { + return Err(malformed("RSDT entry area is not 32-bit aligned")); + } + for entry_offset in (ACPI_HEADER_SIZE..rsdt.length).step_by(4) { + append_add_pointer( + &mut loader, + TABLES_FILE, + TABLES_FILE, + rsdt.offset + .checked_add(entry_offset) + .ok_or_else(|| malformed("RSDT pointer offset overflow"))?, + 4, + )?; + } + append_table_checksum(&mut loader, rsdt)?; + append_add_pointer(&mut loader, RSDP_FILE, TABLES_FILE, 16, 4)?; + append_add_checksum(&mut loader, RSDP_FILE, 8, 0, 20)?; + + if loader.len() > LOADER_BLOB_SIZE { + return Err(malformed("fw_cfg loader exceeds QEMU allocation")); + } + loader.resize(LOADER_BLOB_SIZE, 0); + Ok(AcpiBlobs { + tables, + rsdp, + loader, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_truncated_and_invalid_tables() { + assert!(parse_table_locations(b"DSDT").is_err()); + + let mut short = [0u8; 36]; + short[..4].copy_from_slice(b"DSDT"); + short[4..8].copy_from_slice(&35u32.to_le_bytes()); + assert!(parse_table_locations(&short).is_err()); + + short[4..8].copy_from_slice(&37u32.to_le_bytes()); + assert!(parse_table_locations(&short).is_err()); + } +} diff --git a/dstack/crates/qemu-acpi/src/generated_tables.rs b/dstack/crates/qemu-acpi/src/generated_tables.rs new file mode 100644 index 000000000..5767747e5 --- /dev/null +++ b/dstack/crates/qemu-acpi/src/generated_tables.rs @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// SPDX-License-Identifier: Apache-2.0 + +//! Assembly of the ACPI table blob from generated records only. + +use crate::{ + dsdt, fixed_tables, srat, AcpiBlobs, Compatibility, Error, MachineConfig, TABLE_BLOB_SIZE, +}; + +pub(crate) fn build(config: &MachineConfig) -> Result<AcpiBlobs, Error> { + let mut tables = fixed_tables::facs(); + tables.extend(dsdt::table(config)?); + + let fadt_offset = tables.len() as u32; + tables.extend(fixed_tables::fadt(config.smm, config.cpu_count)); + let madt_offset = tables.len() as u32; + let legacy_irq_overrides = matches!( + config.qemu_version.compatibility(), + Some(Compatibility::V8 | Compatibility::V9Pre92) + ); + tables.extend(fixed_tables::madt( + config.cpu_count, + config.pic, + legacy_irq_overrides, + )); + let srat_offset = config.hugepages.then_some(tables.len() as u32); + if config.hugepages { + tables.extend(srat::build( + config.cpu_count, + config.memory_size, + config.pci_hole64_size, + )); + } + let mcfg_offset = tables.len() as u32; + tables.extend(fixed_tables::mcfg()); + let waet_offset = tables.len() as u32; + tables.extend(fixed_tables::waet()); + + let mut entries = vec![fadt_offset, madt_offset]; + entries.extend(srat_offset); + entries.extend([mcfg_offset, waet_offset]); + tables.extend(fixed_tables::rsdt(&entries)); + tables.resize(tables.len().div_ceil(TABLE_BLOB_SIZE) * TABLE_BLOB_SIZE, 0); + crate::fw_cfg::finish(tables) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::QemuVersion; + + #[test] + fn baseline_matches_qemu_from_scratch() -> Result<(), Error> { + let config = MachineConfig { + qemu_version: QemuVersion::new(11, 1, 0), + cpu_count: 1, + memory_size: 2 << 30, + pic: false, + smm: false, + hugepages: false, + num_gpus: 0, + num_nvswitches: 0, + num_nics: 0, + num_verity_volumes: 0, + hotplug_off: false, + root_verity: true, + pci_hole64_size: None, + }; + let actual = build(&config)?; + let expected = include_bytes!("../fixtures/qemu-11.1-q35-base.bin"); + assert_eq!(&actual.tables[..expected.len()], *expected); + assert!(actual.tables[expected.len()..] + .iter() + .all(|byte| *byte == 0)); + Ok(()) + } +} diff --git a/dstack/crates/qemu-acpi/src/golden_tests.rs b/dstack/crates/qemu-acpi/src/golden_tests.rs new file mode 100644 index 000000000..7a13b4893 --- /dev/null +++ b/dstack/crates/qemu-acpi/src/golden_tests.rs @@ -0,0 +1,307 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(test)] +mod tests { + use crate::{build, Error, MachineConfig, QemuVersion}; + + fn config(nics: u32, volumes: u32) -> MachineConfig { + MachineConfig { + qemu_version: QemuVersion::new(11, 1, 0), + cpu_count: 1, + memory_size: 2 << 30, + pic: false, + smm: false, + hugepages: false, + num_gpus: 0, + num_nvswitches: 0, + num_nics: nics, + num_verity_volumes: volumes, + hotplug_off: false, + root_verity: true, + pci_hole64_size: None, + } + } + + #[test] + fn one_nic_matches_qemu_byte_for_byte() -> Result<(), Error> { + let actual = build(&config(1, 0))?; + assert_eq!( + actual.tables, + include_bytes!("../fixtures/qemu-11.1-q35-one-nic.bin") + ); + assert_eq!( + actual.loader, + include_bytes!("../fixtures/qemu-11.1-q35-one-nic-loader.bin") + ); + assert_eq!( + actual.rsdp, + include_bytes!("../fixtures/qemu-11.1-q35-one-nic-rsdp.bin") + ); + Ok(()) + } + + #[test] + fn numa_loader_and_rsdp_match_qemu_byte_for_byte() -> Result<(), Error> { + let mut numa = config(1, 0); + numa.hugepages = true; + let actual = build(&numa)?; + assert_eq!( + actual.loader, + include_bytes!("../fixtures/qemu-11.1-q35-numa-one-nic-loader.bin") + ); + assert_eq!( + actual.rsdp, + include_bytes!("../fixtures/qemu-11.1-q35-numa-one-nic-rsdp.bin") + ); + Ok(()) + } + + fn qemu_hash(config: MachineConfig) -> Result<String, Error> { + use sha2::{Digest, Sha256}; + + // Keep ownership local so callers can conveniently vary the config. + config.validate()?; + Ok(hex::encode(Sha256::digest(build(&config)?.tables))) + } + + #[test] + fn qemu_version_and_cpu_goldens() -> Result<(), Error> { + let versions = [ + ( + (8, 2, 0), + "ff0c03c7f4026a95b0b2f2e08cef7d501a3a44b4df036c9f26de91828f0215f6", + ), + ( + (9, 1, 0), + "ff0c03c7f4026a95b0b2f2e08cef7d501a3a44b4df036c9f26de91828f0215f6", + ), + ( + (9, 2, 1), + "b183eba66c6e96556f28cefe60ca92620b82488fcefaf66167aad61e2b2e73d1", + ), + ( + (10, 0, 0), + "d0410a8abdbba6d86a19a2eefd491376727e4e2d80c54349d7f30deca14ca6e1", + ), + ( + (11, 0, 0), + "e211dd453e651ef320d28c65d23f578e96468614614e7b1172f18df5052d0f1f", + ), + ( + (11, 1, 0), + "09f99e5dcf36b80a9258e849b3a9a70e7914244008e6a92a689eb22e4cc17a8f", + ), + ]; + for ((major, minor, micro), expected) in versions { + let mut c = config(1, 0); + c.qemu_version = QemuVersion::new(major, minor, micro); + assert_eq!(qemu_hash(c)?, expected); + } + + let cpus = [ + ( + 1, + "09f99e5dcf36b80a9258e849b3a9a70e7914244008e6a92a689eb22e4cc17a8f", + ), + ( + 2, + "1362c6f473801b34f93df4e2966b2d3350f7f9113750da78a2cb799177157da8", + ), + ( + 8, + "0f9fc08c5efd03f1cb3cfaa1a0a09c45dcfbc18183dbc5aa1367ad10eb65135a", + ), + ( + 64, + "089fda95fb5d45ba9a5f46ebc99d85e1331daa4dd38f60597eec9a3d95c94bb5", + ), + ( + 256, + "d1c3291dbdf0ccc4acdecce5f78fbe80e0ca7d6d803a62c9f5b59ee8c7e0a1b5", + ), + ( + 4096, + "6a4bbd704addd10996632d6535432598c742789e60b29d055a7784be927588a5", + ), + ]; + for (count, expected) in cpus { + let mut c = config(1, 0); + c.cpu_count = count; + assert_eq!(qemu_hash(c)?, expected); + } + Ok(()) + } + + #[test] + fn qemu_device_count_goldens() -> Result<(), Error> { + let cases = [ + ( + 0, + 0, + 0, + 0, + "93a140bbc031dc313ed4011b838c8b22ffdecb474435d9fcb294be97194c953c", + ), + ( + 1, + 0, + 0, + 0, + "09f99e5dcf36b80a9258e849b3a9a70e7914244008e6a92a689eb22e4cc17a8f", + ), + ( + 2, + 0, + 0, + 0, + "fbf083121cc3b46b3b7913e45e635e6bb31e11f0cb6e029aa504bc4de68e2d64", + ), + ( + 1, + 1, + 0, + 0, + "fbf083121cc3b46b3b7913e45e635e6bb31e11f0cb6e029aa504bc4de68e2d64", + ), + ( + 1, + 4, + 0, + 0, + "7d722d7e3d811aa7978030af1d2a6c6aa68a5ab9267d7dff4e57babdf1476870", + ), + ( + 1, + 0, + 1, + 0, + "f47ab428541cb334c6de6e59e7fcf44a5db7b314e9dd4c643978968917eb25b2", + ), + ( + 1, + 0, + 8, + 0, + "ae3fefc72eb747cbff363e4f5ac7f3366f257849dc5f2719a9368303abaef4cc", + ), + ( + 1, + 0, + 1, + 1, + "8a48a13bc6041d73f7decce488054a8d25800cc82e11fa9bd1687e010ac9c9b0", + ), + ( + 1, + 0, + 1, + 4, + "2052ea73c74e1462947e600c95742e48cae0f7a84bc0ec79ab12f7a7818aec7a", + ), + ]; + for (nics, volumes, gpus, switches, expected) in cases { + let mut c = config(nics, volumes); + c.num_gpus = gpus; + c.num_nvswitches = switches; + assert_eq!(qemu_hash(c)?, expected); + } + Ok(()) + } + + #[test] + fn qemu_memory_layout_boundaries() -> Result<(), Error> { + let cases = [ + ( + 1, + "8d7620126cfd2a2edabbc8c9f289ca2993f547d595eed61fd508d24f2b52e3ac", + ), + ( + 2049, + "646e5cc620fd7819f8f4756c712619d2a72c29a175fc04b79867cee80e67cf82", + ), + ( + 2815, + "ee35c5e548a3f527dd12b1dd6ee14e093fd6063056470b3ef6cb6c844e4186a4", + ), + ( + 2816, + "3bf181108245994ceb7e983b1fa62009dcd56f7b49fd1e96ef15eb07d04aefc9", + ), + ( + 1_048_576, + "f22a114b0975b18200553442d6c9fab172fb930252c1a95926c594b2c25bca57", + ), + ]; + for (mib, expected) in cases { + let mut c = config(1, 0); + c.cpu_count = 2; + c.memory_size = mib * 1024 * 1024; + c.hugepages = true; + c.num_gpus = 1; + assert_eq!(qemu_hash(c)?, expected); + } + Ok(()) + } + + #[test] + fn pxb_supports_qemus_full_gpu_range() -> Result<(), Error> { + for gpus in [1, 8, 32] { + let mut c = config(1, 0); + c.hugepages = true; + c.num_gpus = gpus; + assert_eq!( + qemu_hash(c)?, + "a8449287b161102ca136f892d13d6bc853d1abdc9ba7804e886f0a42784878c3" + ); + } + Ok(()) + } + + #[test] + fn device_kinds_share_qemus_slot_allocation() -> Result<(), Error> { + assert_eq!(build(&config(5, 0))?.tables, build(&config(1, 4))?.tables); + Ok(()) + } + + #[test] + fn hostile_counts_are_rejected_without_generation() { + let mut c = config(u32::MAX, u32::MAX); + c.cpu_count = u32::MAX; + c.num_gpus = u32::MAX; + c.num_nvswitches = u32::MAX; + assert!(crate::build(&c).is_err()); + + let mut c = config(1, 0); + c.cpu_count = 0; + assert!(crate::build(&c).is_err()); + + let mut c = config(1, 0); + c.memory_size = 0; + assert!(crate::build(&c).is_err()); + } + + #[test] + fn validated_boundary_inputs_generate_safely() -> Result<(), Error> { + for version in [ + QemuVersion::new(8, 0, 0), + QemuVersion::new(9, 1, 0), + QemuVersion::new(9, 2, 0), + QemuVersion::new(10, 0, 0), + QemuVersion::new(11, 0, 0), + QemuVersion::new(11, 1, 0), + ] { + for cpus in [1, 255, 256, 4096] { + for memory_size in [1, 0xafff_ffff, 0xb000_0000, u64::MAX] { + let mut c = config(0, 0); + c.qemu_version = version; + c.cpu_count = cpus; + c.memory_size = memory_size; + c.pci_hole64_size = Some(u64::MAX); + crate::build(&c)?; + } + } + } + Ok(()) + } +} diff --git a/dstack/crates/qemu-acpi/src/lib.rs b/dstack/crates/qemu-acpi/src/lib.rs new file mode 100644 index 000000000..d7c5b0f79 --- /dev/null +++ b/dstack/crates/qemu-acpi/src/lib.rs @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Pure Rust generation of the QEMU Q35 ACPI blobs measured by dstack. +//! +//! This crate intentionally models the observable ACPI ABI rather than a +//! virtual machine. Its output is expected to match QEMU byte for byte for a +//! supported compatibility profile and machine topology. +//! +//! QEMU releases newer than the newest modeled profile are generated with that +//! profile (see [`QemuVersion::compatibility`]) instead of being rejected, so a +//! QEMU upgrade that leaves the ACPI ABI alone keeps working and one that does +//! not surfaces as a blob mismatch the caller can act on. + +mod aml_encode; +mod cpu; +mod dsdt; +mod fixed_tables; +mod fw_cfg; +mod generated_tables; +#[cfg(test)] +mod golden_tests; +mod profile; +mod srat; +mod topology; + +pub use profile::{Compatibility, QemuVersion}; +pub use topology::{MachineConfig, TopologyError}; + +/// Allocation granularity of QEMU's `etc/acpi/tables` fw_cfg blob. +pub const TABLE_BLOB_SIZE: usize = 128 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AcpiBlobs { + pub tables: Vec<u8>, + pub rsdp: Vec<u8>, + pub loader: Vec<u8>, +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error(transparent)] + Topology(#[from] TopologyError), + #[error("no ACPI compatibility profile for QEMU {0}; releases older than 8.0 are not modeled")] + UnsupportedVersion(QemuVersion), + #[error("malformed generated ACPI tables: missing {0}")] + MalformedTables(String), +} + +/// Generate the complete set of fw_cfg blobs measured by dstack. +/// +/// The implementation accepts arbitrary valid counts and sizes; the familiar +/// small matrices used by differential tests are not implementation limits. +pub fn build(config: &MachineConfig) -> Result<AcpiBlobs, Error> { + config.validate()?; + generated_tables::build(config) +} diff --git a/dstack/crates/qemu-acpi/src/profile.rs b/dstack/crates/qemu-acpi/src/profile.rs new file mode 100644 index 000000000..c74e7d43d --- /dev/null +++ b/dstack/crates/qemu-acpi/src/profile.rs @@ -0,0 +1,143 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +use core::fmt; +use core::str::FromStr; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct QemuVersion { + pub major: u32, + pub minor: u32, + pub micro: u32, +} + +impl QemuVersion { + pub const fn new(major: u32, minor: u32, micro: u32) -> Self { + Self { + major, + minor, + micro, + } + } + + /// Map a QEMU version onto the ACPI compatibility profile to generate with. + /// + /// Versions newer than the newest modeled profile fall back to + /// [`Compatibility::LATEST`] rather than failing: most QEMU releases do not + /// change the Q35 ACPI ABI, so extrapolating is right far more often than + /// not, and when it is wrong the generated blobs simply do not match the + /// measured ones. Refusing to generate turns every such deployment into a + /// verification error, which is the same outcome with less information, so + /// the caller is left to decide what a mismatch means. + /// + /// Versions older than 8.0 return `None`: their ABI was never modeled, and + /// clamping *down* to the oldest profile would be a guess in the direction + /// where QEMU's ACPI output is known to differ. + pub const fn compatibility(self) -> Option<Compatibility> { + match (self.major, self.minor) { + (8, _) => Some(Compatibility::V8), + (9, 0..=1) => Some(Compatibility::V9Pre92), + (9, _) => Some(Compatibility::V9_2), + (10, _) => Some(Compatibility::V10), + (11, 0) => Some(Compatibility::V11_0), + (11, 1..) => Some(Compatibility::V11_1), + (12.., _) => Some(Compatibility::LATEST), + _ => None, + } + } +} + +impl fmt::Display for QemuVersion { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}.{}.{}", self.major, self.minor, self.micro) + } +} + +impl FromStr for QemuVersion { + type Err = &'static str; + + fn from_str(value: &str) -> Result<Self, Self::Err> { + let mut parts = value.split('.'); + let major = parts + .next() + .ok_or("missing major")? + .parse() + .map_err(|_| "invalid major")?; + let minor = parts + .next() + .ok_or("missing minor")? + .parse() + .map_err(|_| "invalid minor")?; + let micro = parts + .next() + .ok_or("missing micro")? + .parse() + .map_err(|_| "invalid micro")?; + if parts.next().is_some() { + return Err("too many version components"); + } + Ok(Self::new(major, minor, micro)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Compatibility { + V8, + V9Pre92, + V9_2, + V10, + V11_0, + V11_1, +} + +impl Compatibility { + /// Newest modeled profile, used for QEMU versions released after it. + /// Update this together with every new profile. + pub const LATEST: Self = Self::V11_1; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn versions_map_to_their_own_profile() { + let cases = [ + ((8, 2, 2), Compatibility::V8), + ((9, 1, 0), Compatibility::V9Pre92), + ((9, 2, 1), Compatibility::V9_2), + ((10, 0, 0), Compatibility::V10), + ((11, 0, 3), Compatibility::V11_0), + ((11, 1, 0), Compatibility::V11_1), + ]; + for ((major, minor, micro), expected) in cases { + assert_eq!( + QemuVersion::new(major, minor, micro).compatibility(), + Some(expected), + "{major}.{minor}.{micro}" + ); + } + } + + /// A QEMU release newer than the newest modeled profile must still produce + /// blobs: if its ACPI ABI is unchanged they match, and if it changed the + /// caller sees a digest mismatch instead of a "cannot generate" error. + #[test] + fn versions_newer_than_the_newest_profile_clamp_to_it() { + for version in [ + QemuVersion::new(11, 9, 0), + QemuVersion::new(12, 0, 0), + QemuVersion::new(99, 4, 1), + ] { + assert_eq!(version.compatibility(), Some(Compatibility::LATEST)); + } + } + + #[test] + fn versions_older_than_the_oldest_profile_are_rejected() { + for version in [QemuVersion::new(7, 2, 0), QemuVersion::new(0, 0, 0)] { + assert_eq!(version.compatibility(), None); + } + } +} diff --git a/dstack/crates/qemu-acpi/src/srat.rs b/dstack/crates/qemu-acpi/src/srat.rs new file mode 100644 index 000000000..ceb480a3d --- /dev/null +++ b/dstack/crates/qemu-acpi/src/srat.rs @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// SPDX-License-Identifier: Apache-2.0 + +fn header(length: u32) -> Vec<u8> { + let mut out = b"SRAT".to_vec(); + out.extend_from_slice(&length.to_le_bytes()); + out.extend_from_slice(&[1, 0]); + out.extend_from_slice(b"BOCHS "); + out.extend_from_slice(b"BXPC "); + out.extend_from_slice(&1u32.to_le_bytes()); + out.extend_from_slice(b"BXPC"); + out.extend_from_slice(&1u32.to_le_bytes()); + out.extend_from_slice(&1u32.to_le_bytes()); + out.extend_from_slice(&0u64.to_le_bytes()); + out +} + +fn memory_affinity(base: u64, length: u64, enabled: bool) -> Vec<u8> { + let mut out = vec![1, 40]; + out.extend_from_slice(&0u32.to_le_bytes()); // proximity domain + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&base.to_le_bytes()); + out.extend_from_slice(&length.to_le_bytes()); + out.extend_from_slice(&0u32.to_le_bytes()); + out.extend_from_slice(&(enabled as u32).to_le_bytes()); + out.extend_from_slice(&0u64.to_le_bytes()); + out +} + +pub(crate) fn build(cpu_count: u32, memory_size: u64, pci_hole64_size: Option<u64>) -> Vec<u8> { + let mut body = Vec::new(); + for index in 0..cpu_count { + if index < 255 { + body.extend_from_slice(&[0, 16, 0, index as u8]); + body.extend_from_slice(&1u32.to_le_bytes()); + body.extend_from_slice(&[0, 0, 0, 0]); + body.extend_from_slice(&0u32.to_le_bytes()); + } else { + body.extend_from_slice(&[2, 24]); + body.extend_from_slice(&0u16.to_le_bytes()); + body.extend_from_slice(&0u32.to_le_bytes()); + body.extend_from_slice(&index.to_le_bytes()); + body.extend_from_slice(&1u32.to_le_bytes()); + body.extend_from_slice(&0u32.to_le_bytes()); + body.extend_from_slice(&0u32.to_le_bytes()); + } + } + let low = if memory_size >= 0xb000_0000 { + 0x8000_0000 + } else { + memory_size + }; + body.extend_from_slice(&memory_affinity(0, low.min(0xa_0000), true)); + if low > 0x10_0000 { + body.extend_from_slice(&memory_affinity(0x10_0000, low - 0x10_0000, true)); + } else { + body.extend_from_slice(&memory_affinity(0, 0, false)); + } + if memory_size > low { + let high_length = memory_size - low; + let high_end = 0x1_0000_0000u64.saturating_add(high_length); + // qemu64 is an AMD CPU model. QEMU relocates RAM above 1 TiB when the + // rounded end of RAM plus the Q35 64-bit PCI hole reaches AMD's + // reserved HyperTransport range (pc_max_used_gpa/pc_memory_init). + let pci_hole_start = high_end.saturating_add((1 << 30) - 1) & !((1 << 30) - 1); + let pci_hole_size = pci_hole64_size.unwrap_or(1 << 35); + let max_used = pci_hole_start + .saturating_add(pci_hole_size) + .saturating_sub(1); + let high_base = if max_used >= 0xfd_0000_0000 { + 0x100_0000_0000 + } else { + 0x1_0000_0000 + }; + body.extend_from_slice(&memory_affinity(high_base, high_length, true)); + } else { + body.extend_from_slice(&memory_affinity(0, 0, false)); + } + let mut out = header((48 + body.len()) as u32); + out.extend_from_slice(&body); + out +} diff --git a/dstack/crates/qemu-acpi/src/topology.rs b/dstack/crates/qemu-acpi/src/topology.rs new file mode 100644 index 000000000..a4db35d15 --- /dev/null +++ b/dstack/crates/qemu-acpi/src/topology.rs @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +use crate::QemuVersion; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MachineConfig { + pub qemu_version: QemuVersion, + pub cpu_count: u32, + pub memory_size: u64, + pub pic: bool, + pub smm: bool, + pub hugepages: bool, + pub num_gpus: u32, + pub num_nvswitches: u32, + pub num_nics: u32, + pub num_verity_volumes: u32, + pub hotplug_off: bool, + pub root_verity: bool, + pub pci_hole64_size: Option<u64>, +} + +#[derive(Debug, thiserror::Error)] +pub enum TopologyError { + #[error("cpu_count must be greater than zero")] + NoCpus, + #[error("memory_size must be greater than zero")] + NoMemory, + #[error("cpu_count exceeds the Q35 limit of 4096: {0}")] + TooManyCpus(u32), + #[error("the requested topology needs {requested} devices on the root PCIe bus, but QEMU has only {available} free slots")] + TooManyRootBusDevices { requested: i64, available: i64 }, + #[error("the requested topology needs {requested} GPU root ports on the PXB, but QEMU supports at most 32")] + TooManyPxbPorts { requested: u32 }, + #[error("the requested topology places {requested} devices before the fixed-address PXB, but QEMU has only {available} slots there")] + TooManyPrePxbDevices { requested: u64, available: u64 }, + #[error("NVSwitch passthrough requires an iommufd object, but no GPU creates one")] + NvswitchWithoutIommufd, +} + +impl MachineConfig { + pub fn validate(&self) -> Result<(), TopologyError> { + if self.cpu_count == 0 { + return Err(TopologyError::NoCpus); + } + if self.memory_size == 0 { + return Err(TopologyError::NoMemory); + } + if self.cpu_count > 4096 { + return Err(TopologyError::TooManyCpus(self.cpu_count)); + } + if self.hugepages && self.num_gpus > 32 { + return Err(TopologyError::TooManyPxbPorts { + requested: self.num_gpus, + }); + } + if self.hugepages && self.num_gpus > 0 { + let requested = 4 + + u64::from(self.root_verity) + + u64::from(self.num_nics) + + u64::from(self.num_verity_volumes); + if requested > 16 { + return Err(TopologyError::TooManyPrePxbDevices { + requested, + available: 16, + }); + } + } + let fixed_delta = i64::from(self.root_verity) - 1; + let passthrough_ports = if self.hugepages && self.num_gpus > 0 { + i64::from(self.num_nvswitches) + } else { + i64::from(self.num_gpus) + i64::from(self.num_nvswitches) + }; + let requested = i64::from(self.num_nics) + + i64::from(self.num_verity_volumes) + + fixed_delta + + passthrough_ports; + let available = if self.hugepages && self.num_gpus > 0 { + 25 + } else { + 26 + }; + if requested > available { + return Err(TopologyError::TooManyRootBusDevices { + requested, + available, + }); + } + if self.num_nvswitches > 0 && self.num_gpus == 0 { + return Err(TopologyError::NvswitchWithoutIommufd); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn pxb_config() -> MachineConfig { + MachineConfig { + qemu_version: QemuVersion::new(11, 1, 0), + cpu_count: 1, + memory_size: 2 << 30, + pic: false, + smm: false, + hugepages: true, + num_gpus: 1, + num_nvswitches: 0, + num_nics: 1, + num_verity_volumes: 0, + hotplug_off: false, + root_verity: true, + pci_hole64_size: None, + } + } + + #[test] + fn fixed_address_pxb_slot_must_remain_free() { + let mut config = pxb_config(); + config.num_nics = 11; + assert!(config.validate().is_ok()); + + config.num_nics = 12; + assert!(matches!( + config.validate(), + Err(TopologyError::TooManyPrePxbDevices { + requested: 17, + available: 16 + }) + )); + } +} diff --git a/dstack/ct_monitor/src/main.rs b/dstack/ct_monitor/src/main.rs index dd5d9550d..5ddc2e2c1 100644 --- a/dstack/ct_monitor/src/main.rs +++ b/dstack/ct_monitor/src/main.rs @@ -42,7 +42,6 @@ struct VerificationRequest { quote: String, event_log: String, vm_config: String, - pccs_url: Option<String>, } /// Response from dstack-verifier @@ -88,7 +87,6 @@ struct AcmeInfoResponse { struct Monitor { gateway_uri: String, verifier_url: String, - pccs_url: Option<String>, base_domain: String, known_keys: BTreeSet<Vec<u8>>, last_checked: Option<u64>, @@ -112,13 +110,12 @@ struct CTLog { impl Monitor { /// Create a new monitor /// `gateway` format: `base_domain[:port]`, e.g., `example.com` or `example.com:8443` - fn new(gateway: String, verifier_url: String, pccs_url: Option<String>) -> Result<Self> { + fn new(gateway: String, verifier_url: String) -> Result<Self> { let (base_domain, gateway_uri) = Self::parse_gateway(&gateway)?; validate_domain(&base_domain)?; Ok(Self { gateway_uri, verifier_url, - pccs_url, base_domain, known_keys: BTreeSet::new(), last_checked: None, @@ -175,7 +172,6 @@ impl Monitor { quote: hex::encode("e_response.quote), event_log: quote_response.event_log, vm_config: quote_response.vm_config, - pccs_url: self.pccs_url.clone(), }; // Call verifier @@ -402,10 +398,6 @@ struct Args { /// The dstack-verifier URL #[arg(short, long, env = "VERIFIER_URL")] verifier_url: String, - - /// PCCS URL for TDX collateral fetching (optional) - #[arg(long, env = "PCCS_URL")] - pccs_url: Option<String>, } #[tokio::main] @@ -416,7 +408,7 @@ async fn main() -> anyhow::Result<()> { fmt().with_env_filter(filter).with_ansi(false).init(); } let args = Args::parse(); - let mut monitor = Monitor::new(args.gateway, args.verifier_url, args.pccs_url)?; + let mut monitor = Monitor::new(args.gateway, args.verifier_url)?; monitor.run().await; Ok(()) } diff --git a/dstack/dstack-attest/Cargo.toml b/dstack/dstack-attest/Cargo.toml index 57c37a0c3..eed37c668 100644 --- a/dstack/dstack-attest/Cargo.toml +++ b/dstack/dstack-attest/Cargo.toml @@ -17,9 +17,12 @@ dcap-qvl.workspace = true dstack-types.workspace = true ez-hash.workspace = true fs-err.workspace = true +safe-write.workspace = true +rustix = { workspace = true, features = ["process"] } hex.workspace = true hex_fmt.workspace = true or-panic.workspace = true +pem.workspace = true scale = { workspace = true, features = ["derive"] } sev-snp-attest.workspace = true sev-snp-qvl.workspace = true @@ -35,6 +38,7 @@ nsm-qvl.workspace = true tpm-qvl.workspace = true tpm-types.workspace = true tracing.workspace = true +x509-parser.workspace = true insta.workspace = true errify.workspace = true aws-nitro-enclaves-nsm-api = { version = "0.4", optional = true } @@ -45,6 +49,9 @@ rsa = { workspace = true, optional = true } tpm2 = { workspace = true, optional = true } [features] +# Synthetic-attestation helpers used to build fixtures. Simulator only -- see +# the gated `impl Attestation` in src/v1.rs for why production must not have it. +simulator = [] quote = [ "aws-nitro-enclaves-nsm-api", "ciborium", @@ -58,3 +65,5 @@ quote = [ futures = { workspace = true } tokio = { workspace = true, features = ["full"] } dstack-mr = { workspace = true } +rcgen = { workspace = true } +tempfile = { workspace = true } diff --git a/dstack/dstack-attest/src/attestation.rs b/dstack/dstack-attest/src/attestation.rs index 1fb8813c9..e55d63827 100644 --- a/dstack/dstack-attest/src/attestation.rs +++ b/dstack/dstack-attest/src/attestation.rs @@ -12,12 +12,13 @@ pub const TDX_QUOTE_REPORT_DATA_RANGE: std::ops::Range<usize> = 568..632; use std::{borrow::Cow, time::SystemTime}; use anyhow::{anyhow, bail, Context, Result}; -use cc_eventlog::{RuntimeEvent, TdxEvent}; +use cc_eventlog::{EventLogVersion, RuntimeEvent, TdxEvent}; use dcap_qvl::{ collateral::CollateralClient, quote::{EnclaveReport, Quote, Report, TDReport10, TDReport15}, verify::VerifiedReport as TdxVerifiedReport, }; +pub use dstack_types::CollateralUrls; #[cfg(feature = "quote")] use dstack_types::SysConfig; use dstack_types::{mr_config::MrConfigV3, KeyProviderInfo, Platform, VmConfig}; @@ -29,30 +30,214 @@ use serde_human_bytes as hex_bytes; use sha2::Digest as _; use tpm_qvl::verify::VerifiedReport as TpmVerifiedReport; +/// File paths for attestation trust anchors. Empty fields retain the vendor +/// production roots. Paths are read by the verifier, never by the attester. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RootCaPaths { + pub tdx: Option<std::path::PathBuf>, + pub gcp_tpm: Option<std::path::PathBuf>, + pub aws_nitro_enclave: Option<std::path::PathBuf>, + pub aws_nitro_tpm: Option<std::path::PathBuf>, + pub sev_snp_milan: Option<std::path::PathBuf>, + pub sev_snp_genoa: Option<std::path::PathBuf>, + pub sev_snp_turin: Option<std::path::PathBuf>, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AttestationVerifierConfig { + #[serde(default)] + pub insecure_allow_external_trust_anchors: bool, + #[serde(default)] + pub urls: CollateralUrls, + #[serde(default)] + pub root_ca: RootCaPaths, +} + +pub struct AttestationVerifier { + tdx: dcap_qvl::verify::QuoteVerifier, + tdx_collateral: CollateralClient, + gcp_tpm: tpm_qvl::QuoteVerifier, + aws_nitro_enclave: nsm_qvl::QuoteVerifier, + aws_nitro_tpm: nsm_qvl::QuoteVerifier, + sev_snp: sev_snp_qvl::QuoteVerifier, + amd_kds: AmdKdsClient, +} + +impl AttestationVerifier { + pub fn load(config: &AttestationVerifierConfig) -> Result<Self> { + let roots = &config.root_ca; + let RootCaPaths { + tdx, + gcp_tpm, + aws_nitro_enclave, + aws_nitro_tpm, + sev_snp_milan, + sev_snp_genoa, + sev_snp_turin, + } = roots; + let external_requested = [ + tdx, + gcp_tpm, + aws_nitro_enclave, + aws_nitro_tpm, + sev_snp_milan, + sev_snp_genoa, + sev_snp_turin, + ] + .into_iter() + .any(Option::is_some); + anyhow::ensure!( + !external_requested || config.insecure_allow_external_trust_anchors, + "external attestation trust anchors are configured but \ + insecure_allow_external_trust_anchors is false" + ); + let tdx = match read_root_file(tdx.as_deref(), "TDX")? { + Some(root) => dcap_qvl::verify::QuoteVerifier::new(tdx_root_der(root)?), + None => dcap_qvl::verify::QuoteVerifier::new_prod(), + }; + let gcp_tpm = match read_root_file(gcp_tpm.as_deref(), "GCP TPM")? { + Some(root) => tpm_qvl::QuoteVerifier::new(validated_pem_string(root, "GCP TPM")?), + None => tpm_qvl::QuoteVerifier::new_prod(Platform::Gcp)?, + }; + let nsm = |path, name| -> Result<nsm_qvl::QuoteVerifier> { + Ok(match read_root_file(path, name)? { + Some(root) => nsm_qvl::QuoteVerifier::new(validated_pem_string(root, name)?), + None => nsm_qvl::QuoteVerifier::new_prod(), + }) + }; + let mut sev_snp = sev_snp_qvl::QuoteVerifier::new_prod(); + for (path, name, product) in [ + ( + sev_snp_milan.as_deref(), + "SEV-SNP Milan", + sev_snp_qvl::AmdSnpProduct::Milan, + ), + ( + sev_snp_genoa.as_deref(), + "SEV-SNP Genoa", + sev_snp_qvl::AmdSnpProduct::Genoa, + ), + ( + sev_snp_turin.as_deref(), + "SEV-SNP Turin", + sev_snp_qvl::AmdSnpProduct::Turin, + ), + ] { + if let Some(root) = read_root_file(path, name)? { + validate_x509_certificate(&root, name)?; + sev_snp = sev_snp.with_root(product, root); + } + } + let pccs = config + .urls + .pccs + .as_deref() + .filter(|v| !v.trim().is_empty()) + .unwrap_or(dcap_qvl::collateral::PHALA_PCCS_URL); + let amd_kds = config + .urls + .amd_kds + .as_deref() + .filter(|v| !v.trim().is_empty()) + .unwrap_or(sev_snp_qvl::AMD_KDS_DEFAULT_BASE_URL); + Ok(Self { + tdx, + tdx_collateral: CollateralClient::with_default_http(pccs)?, + gcp_tpm, + aws_nitro_enclave: nsm(aws_nitro_enclave.as_deref(), "AWS Nitro Enclave")?, + aws_nitro_tpm: nsm(aws_nitro_tpm.as_deref(), "AWS NitroTPM")?, + sev_snp, + amd_kds: AmdKdsClient::with_base_url(amd_kds)?, + }) + } + + pub fn new_prod(collateral_urls: Option<&CollateralUrls>) -> Result<Self> { + let collateral_urls = collateral_urls.cloned().unwrap_or_default(); + Ok(Self { + tdx: dcap_qvl::verify::QuoteVerifier::new_prod(), + tdx_collateral: CollateralClient::with_default_http( + collateral_urls + .pccs + .as_deref() + .filter(|url| !url.trim().is_empty()) + .unwrap_or(dcap_qvl::collateral::PHALA_PCCS_URL), + )?, + gcp_tpm: tpm_qvl::QuoteVerifier::new_prod(Platform::Gcp)?, + aws_nitro_enclave: nsm_qvl::QuoteVerifier::new_prod(), + aws_nitro_tpm: nsm_qvl::QuoteVerifier::new_prod(), + sev_snp: sev_snp_qvl::QuoteVerifier::new_prod(), + amd_kds: AmdKdsClient::with_base_url( + collateral_urls + .amd_kds + .as_deref() + .filter(|url| !url.trim().is_empty()) + .unwrap_or(sev_snp_qvl::AMD_KDS_DEFAULT_BASE_URL), + )?, + }) + } + + async fn verify_tdx_quote(&self, quote: &[u8]) -> Result<TdxVerifiedReport> { + let collateral = self.tdx_collateral.fetch(quote).await?; + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .context("system clock is before UNIX epoch")? + .as_secs(); + self.tdx.verify(quote, &collateral, now) + } +} + +fn read_root_file(path: Option<&std::path::Path>, platform: &str) -> Result<Option<Vec<u8>>> { + let Some(path) = path else { + return Ok(None); + }; + fs_err::read(path) + .with_context(|| format!("failed to read {platform} root CA from {}", path.display())) + .map(Some) +} + +fn validated_pem_string(root: Vec<u8>, platform: &str) -> Result<String> { + validate_x509_certificate(&root, platform)?; + String::from_utf8(root).with_context(|| format!("{platform} root CA is not UTF-8 PEM")) +} + +fn validate_x509_certificate(root: &[u8], platform: &str) -> Result<()> { + use x509_parser::prelude::{FromDer, X509Certificate}; + let der = if root.starts_with(b"-----BEGIN") { + pem::parse(root) + .with_context(|| format!("failed to parse {platform} root CA PEM"))? + .into_contents() + } else { + root.to_vec() + }; + let (remaining, certificate) = X509Certificate::from_der(&der) + .with_context(|| format!("failed to parse {platform} root CA certificate"))?; + anyhow::ensure!(remaining.is_empty(), "trailing data in {platform} root CA"); + anyhow::ensure!( + certificate.is_ca(), + "{platform} root certificate is not a CA" + ); + Ok(()) +} + +fn tdx_root_der(root: Vec<u8>) -> Result<Vec<u8>> { + validate_x509_certificate(&root, "TDX")?; + if root.starts_with(b"-----BEGIN") { + return Ok(pem::parse(root)?.into_contents()); + } + Ok(root) +} + // Re-export TpmQuote from tpm-types pub use tpm_types::TpmQuote; use crate::amd_sev_snp::{AmdKdsClient, VerifiedAmdSnpReport}; -use crate::v1::{ - is_tdx_acpi_data_event, strip_tdx_event_log_for_config, strip_tdx_runtime_event_log, -}; +use crate::v1::{strip_tdx_event_log_for_config, strip_tdx_runtime_event_log}; pub use crate::v1::{Attestation as AttestationV1, PlatformEvidence, StackEvidence}; pub const SNP_REPORT_DATA_RANGE: std::ops::Range<usize> = 0x50..0x90; -const DSTACK_TDX: &str = "dstack-tdx"; -const DSTACK_AMD_SEV_SNP: &str = "dstack-amd-sev-snp"; -const DSTACK_GCP_TDX: &str = "dstack-gcp-tdx"; -const DSTACK_NITRO_ENCLAVE: &str = "dstack-nitro-enclave"; -const DSTACK_AWS_NITRO_TPM: &str = "dstack-aws-nitro-tpm"; - -fn dcap_collateral_client(pccs_url: Option<&str>) -> Result<CollateralClient> { - match pccs_url.map(str::trim).filter(|url| !url.is_empty()) { - Some(pccs_url) => CollateralClient::with_default_http(pccs_url), - None => CollateralClient::from_env(), - } -} - /// Path to sys-config.json in the host-shared dir. /// /// Honors `DSTACK_HOST_SHARED_DIR` (exported by `dstack-util setup` because the @@ -69,8 +254,9 @@ static QUOTE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// Read vm_config from sys-config.json #[cfg(feature = "quote")] -fn read_vm_config() -> Result<String> { - let content = match fs_err::read_to_string(sys_config_path()) { +fn read_vm_config(path: Option<&std::path::Path>) -> Result<String> { + let path = path.map_or_else(sys_config_path, std::path::Path::to_path_buf); + let content = match fs_err::read_to_string(path) { Ok(content) => content, Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(String::new()), Err(err) => return Err(err).context("Failed to read sys-config"), @@ -86,8 +272,9 @@ fn read_vm_config() -> Result<String> { /// where `mr_config` lives (top-level field, falling back to the one embedded /// in `vm_config`). #[cfg(feature = "quote")] -fn read_mr_config_document() -> Result<Option<String>> { - let content = match fs_err::read_to_string(sys_config_path()) { +fn read_mr_config_document(path: Option<&std::path::Path>) -> Result<Option<String>> { + let path = path.map_or_else(sys_config_path, std::path::Path::to_path_buf); + let content = match fs_err::read_to_string(path) { Ok(content) => content, Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(err) => return Err(err).context("Failed to read sys-config"), @@ -207,6 +394,19 @@ fn find_event_payload(runtime_events: &[RuntimeEvent], event: &str) -> Result<Ve find_event(runtime_events, event).map(|event| event.payload) } +/// Returns ordered payloads for matching boot-time events. +/// +/// Events after `system-ready` are application-controlled and intentionally +/// excluded from system measurements exposed through decoded app info. +fn find_event_payloads(runtime_events: &[RuntimeEvent], name: &str) -> Vec<Vec<u8>> { + runtime_events + .iter() + .take_while(|event| event.event != "system-ready") + .filter(|event| event.event == name) + .map(|event| event.payload.clone()) + .collect() +} + fn decode_vm_config_with_fallback(config: &str, fallback_config: &str) -> Result<VmConfig> { let config = if config.is_empty() { fallback_config @@ -253,27 +453,7 @@ fn mr_config_document_from_config(config: &str) -> Result<Option<String>> { mr_config_document_from_value(&vm_config) } -/// Attestation mode -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Encode, Decode, Serialize, Deserialize)] -pub enum AttestationMode { - /// Intel TDX with DCAP quote only - #[default] - #[serde(rename = "dstack-tdx")] - DstackTdx, - /// GCP TDX with DCAP quote only - #[serde(rename = "dstack-gcp-tdx")] - DstackGcpTdx, - /// Dstack attestation SDK in AWS Nitro Enclave - #[serde(rename = "dstack-nitro-enclave")] - DstackNitroEnclave, - /// AMD SEV-SNP report generated by the dstack attestation SDK. - #[serde(rename = "dstack-amd-sev-snp")] - DstackAmdSevSnp, - /// AWS EC2 NitroTPM attestation document generated by the dstack attestation SDK. - /// Keep this last to preserve SCALE discriminants for existing variants. - #[serde(rename = "dstack-aws-nitro-tpm")] - DstackAwsNitroTpm, -} +pub use dstack_types::TeeVariant; #[cfg(feature = "quote")] fn has_sev_snp_tsm_provider() -> bool { @@ -285,77 +465,40 @@ fn has_sev_snp_tsm_provider() -> bool { false } -fn choose_dstack_attestation_mode(has_tdx: bool, has_sev_snp: bool) -> Result<AttestationMode> { +fn choose_dstack_tee_variant(has_tdx: bool, has_sev_snp: bool) -> Result<TeeVariant> { if has_tdx { - return Ok(AttestationMode::DstackTdx); + return Ok(TeeVariant::DstackTdx); } if has_sev_snp { - return Ok(AttestationMode::DstackAmdSevSnp); + return Ok(TeeVariant::DstackAmdSevSnp); } bail!("Unsupported platform: Dstack(-tdx/-amd-sev-snp)"); } -impl AttestationMode { - /// Detect attestation mode from system - pub fn detect() -> Result<Self> { - let has_tdx = tdx_attest::is_tdx_available(); - let has_sev_snp = - std::path::Path::new("/dev/sev-guest").exists() || has_sev_snp_tsm_provider(); - - // First, try to detect platform from DMI product name - let platform = Platform::detect_or_dstack(); - match platform { - Platform::Dstack => choose_dstack_attestation_mode(has_tdx, has_sev_snp), - Platform::Gcp => { - // GCP platform: TDX + TPM dual mode - if has_tdx { - return Ok(Self::DstackGcpTdx); - } - bail!("Unsupported platform: GCP(-tdx)"); - } - Platform::NitroEnclave => Ok(Self::DstackNitroEnclave), - Platform::AwsEc2 => { - if std::path::Path::new("/dev/tpmrm0").exists() - || std::path::Path::new("/dev/tpm0").exists() - { - return Ok(Self::DstackAwsNitroTpm); - } - bail!("unsupported platform: AWS EC2 without NitroTPM"); - } - } - } - - /// Check if TDX quote should be included - pub fn has_tdx(&self) -> bool { - match self { - Self::DstackTdx => true, - Self::DstackAmdSevSnp => false, - Self::DstackGcpTdx => true, - Self::DstackNitroEnclave => false, - Self::DstackAwsNitroTpm => false, - } - } +/// Detect the attestation variant exposed by the current guest environment. +pub fn detect_tee_variant() -> Result<TeeVariant> { + let has_tdx = tdx_attest::is_tdx_available(); + let has_sev_snp = std::path::Path::new("/dev/sev-guest").exists() || has_sev_snp_tsm_provider(); - /// Single TPM PCR used for all dstack measured events (TDX RTMR3 analogue). - /// - /// AWS NitroTPM uses non-resettable SHA384 PCR14. GCP uses SHA256 PCR14. - /// There is no separate runtime PCR (PCR23 is not used). - pub fn tpm_event_pcr_and_bank(&self) -> Option<(u32, &'static str)> { - match self { - Self::DstackGcpTdx => Some((14, "sha256")), - Self::DstackAwsNitroTpm => Some((u32::from(AWS_NITRO_TPM_EVENT_PCR), "sha384")), - Self::DstackTdx | Self::DstackAmdSevSnp | Self::DstackNitroEnclave => None, + // First, try to detect platform from DMI product name + let platform = Platform::detect_or_dstack(); + match platform { + Platform::Dstack => choose_dstack_tee_variant(has_tdx, has_sev_snp), + Platform::Gcp => { + // GCP platform: TDX + TPM dual mode + if has_tdx { + return Ok(TeeVariant::DstackGcpTdx); + } + bail!("Unsupported platform: GCP(-tdx)"); } - } - - /// As string for debug - pub fn as_str(&self) -> &'static str { - match self { - Self::DstackTdx => DSTACK_TDX, - Self::DstackAmdSevSnp => DSTACK_AMD_SEV_SNP, - Self::DstackGcpTdx => DSTACK_GCP_TDX, - Self::DstackNitroEnclave => DSTACK_NITRO_ENCLAVE, - Self::DstackAwsNitroTpm => DSTACK_AWS_NITRO_TPM, + Platform::NitroEnclave => Ok(TeeVariant::DstackNitroEnclave), + Platform::AwsEc2 => { + if std::path::Path::new("/dev/tpmrm0").exists() + || std::path::Path::new("/dev/tpm0").exists() + { + return Ok(TeeVariant::DstackAwsNitroTpm); + } + bail!("unsupported platform: AWS EC2 without NitroTPM"); } } } @@ -608,8 +751,15 @@ impl VersionedAttestation { bail!("Empty attestation bytes"); }; if first == 0x00 { - let legacy = LegacyVersionedAttestation::decode(&mut &bytes[..]) + let mut input = bytes; + let legacy = LegacyVersionedAttestation::decode(&mut input) .context("Failed to decode legacy VersionedAttestation")?; + if !input.is_empty() { + bail!( + "Trailing bytes after legacy VersionedAttestation: {}", + input.len() + ); + } return match legacy { LegacyVersionedAttestation::V0 { attestation } => Ok(Self::V0 { attestation }), }; @@ -689,8 +839,11 @@ pub trait TdxAttestationExt { /// Returns the TDX event log serialized as JSON. fn tdx_event_log_string(&self) -> Option<String> { - self.tdx_event_log() - .map(|event_log| serde_json::to_string(event_log).unwrap_or_default()) + self.tdx_event_log().map(|event_log| { + let mut events: Vec<TdxEvent> = event_log.to_vec(); + cc_eventlog::tdx::fill_v2_preimages(&mut events); + serde_json::to_string(&events).unwrap_or_default() + }) } /// Returns the parsed TD10 report from the embedded TDX quote. @@ -715,6 +868,40 @@ impl TdxAttestationExt for AttestationV1 { } impl AttestationV1 { + /// Convert a V1 dstack attestation back to the legacy SCALE schema. + /// + /// This is only lossless for the original dstack stack with V1 runtime + /// events. Pod payloads and newer event encodings must remain on the V1 + /// msgpack wire format. + pub fn try_into_legacy(self) -> Result<Attestation> { + let Self { + platform, stack, .. + } = self; + let StackEvidence::Dstack { + report_data, + runtime_events, + config, + } = stack + else { + bail!("dstack-pod attestation cannot be represented by the legacy schema"); + }; + if runtime_events + .iter() + .any(|event| !matches!(event.version, EventLogVersion::V1)) + { + bail!("non-V1 runtime events cannot be represented by the legacy schema"); + } + Ok(Attestation { + quote: platform_into_legacy_quote(platform), + runtime_events, + report_data: report_data + .try_into() + .map_err(|_| anyhow!("stack.report_data must be 64 bytes"))?, + config, + report: (), + }) + } + /// Decode the VM config from the external or embedded config. pub fn decode_vm_config<'a>(&'a self, config: &'a str) -> Result<VmConfig> { decode_vm_config_with_fallback(config, self.stack.config()) @@ -761,6 +948,7 @@ impl AttestationV1 { key_provider_info, os_image_hash, compose_hash, + init_script_hashes: Some(find_event_payloads(runtime_events, "init-script-hash")), } }; @@ -836,31 +1024,14 @@ impl AttestationV1 { } } - /// Verify the quote with optional custom time (testing hook). - pub async fn verify_with_time( - self, - pccs_url: Option<&str>, - now: Option<SystemTime>, - ) -> Result<VerifiedAttestation> { - self.verify_with_time_with_amd_kds_client(pccs_url, now, None) - .await - } - - /// Verify the quote with a caller-owned AMD KDS client. - pub async fn verify_with_amd_kds_client( - self, - pccs_url: Option<&str>, - amd_kds_client: &AmdKdsClient, - ) -> Result<VerifiedAttestation> { - self.verify_with_time_with_amd_kds_client(pccs_url, None, Some(amd_kds_client)) - .await + pub async fn verify(self, verifier: &AttestationVerifier) -> Result<VerifiedAttestation> { + self.verify_with_time(verifier, None).await } - async fn verify_with_time_with_amd_kds_client( + pub async fn verify_with_time( self, - pccs_url: Option<&str>, + verifier: &AttestationVerifier, now: Option<SystemTime>, - amd_kds_client: Option<&AmdKdsClient>, ) -> Result<VerifiedAttestation> { let AttestationV1 { version: _, @@ -898,16 +1069,18 @@ impl AttestationV1 { }; let report = match &platform { PlatformEvidence::Tdx { quote, .. } => DstackVerifiedReport::DstackTdx( - verify_tdx_quote_with_events(pccs_url, quote, &runtime_events, &report_data) + verify_tdx_quote_with_events(verifier, quote, &runtime_events, &report_data) .await?, ), PlatformEvidence::GcpTdx { quote, tpm_quote, .. } => { let tdx_report = - verify_tdx_quote_with_events(pccs_url, quote, &runtime_events, &report_data) + verify_tdx_quote_with_events(verifier, quote, &runtime_events, &report_data) .await?; - let tpm_report = tpm_qvl::get_collateral_and_verify(tpm_quote) + let tpm_report = verifier + .gcp_tpm + .fetch_and_verify(tpm_quote) .await .context("failed to verify TPM quote")?; let qualifying_data = sha256(quote); @@ -935,13 +1108,10 @@ impl AttestationV1 { let nsm = DstackNitroQuote { nsm_quote: nsm_quote.clone(), }; - let verified_report = nsm_qvl::verify_attestation( - &nsm.nsm_quote, - nsm_qvl::AWS_NITRO_ENCLAVES_ROOT_G1, - None, - now, - ) - .context("NSM attestation verification failed")?; + let verified_report = verifier + .aws_nitro_enclave + .verify(&nsm.nsm_quote, None, now) + .context("NSM attestation verification failed")?; let Some(user_data) = verified_report.user_data.clone() else { bail!("NSM attestation document does not contain user_data"); }; @@ -962,6 +1132,7 @@ impl AttestationV1 { } PlatformEvidence::AwsNitroTpm { attestation_doc } => { let verified_report = verify_aws_nitro_tpm_attestation_doc( + verifier, attestation_doc, &runtime_events, &report_data, @@ -975,22 +1146,24 @@ impl AttestationV1 { cert_chain, mr_config, } => { - let owned_kds_client; - let kds_client = match amd_kds_client { - Some(client) => client, - None => { - owned_kds_client = AmdKdsClient::new()?; - &owned_kds_client - } - }; - let verified = kds_client - .verify_evidence_with_kds_fallback(report, cert_chain, &report_data) + let verified = verifier + .sev_snp + .fetch_and_verify(&verifier.amd_kds, report, cert_chain, &report_data) .await?; verify_snp_mr_config_host_data(mr_config, &verified.host_data)?; DstackVerifiedReport::DstackAmdSevSnp(verified) } }; + match &platform { + PlatformEvidence::Tdx { event_log, .. } + | PlatformEvidence::GcpTdx { event_log, .. } => { + cc_eventlog::tdx::validate_v2_preimages(event_log) + .context("Failed to validate TDX V2 event digest preimages")?; + } + _ => {} + } + Ok(VerifiedAttestation { quote: platform_into_legacy_quote(platform), runtime_events, @@ -1004,18 +1177,13 @@ impl AttestationV1 { pub async fn verify_with_ra_pubkey( self, ra_pubkey_der: &[u8], - pccs_url: Option<&str>, + verifier: &AttestationVerifier, ) -> Result<VerifiedAttestation> { let expected_report_data = QuoteContentType::RaTlsCert.to_report_data(ra_pubkey_der); if self.report_data()? != expected_report_data { bail!("report data mismatch"); } - self.verify(pccs_url).await - } - - /// Verify the quote. - pub async fn verify(self, pccs_url: Option<&str>) -> Result<VerifiedAttestation> { - self.verify_with_time(pccs_url, None).await + self.verify(verifier).await } } @@ -1187,13 +1355,13 @@ pub enum AttestationQuote { } impl AttestationQuote { - pub fn mode(&self) -> AttestationMode { + pub fn variant(&self) -> TeeVariant { match self { - AttestationQuote::DstackTdx(_) => AttestationMode::DstackTdx, - AttestationQuote::DstackAmdSevSnp(_) => AttestationMode::DstackAmdSevSnp, - AttestationQuote::DstackGcpTdx(_) => AttestationMode::DstackGcpTdx, - AttestationQuote::DstackNitroEnclave(_) => AttestationMode::DstackNitroEnclave, - AttestationQuote::DstackAwsNitroTpm(_) => AttestationMode::DstackAwsNitroTpm, + AttestationQuote::DstackTdx(_) => TeeVariant::DstackTdx, + AttestationQuote::DstackAmdSevSnp(_) => TeeVariant::DstackAmdSevSnp, + AttestationQuote::DstackGcpTdx(_) => TeeVariant::DstackGcpTdx, + AttestationQuote::DstackNitroEnclave(_) => TeeVariant::DstackNitroEnclave, + AttestationQuote::DstackAwsNitroTpm(_) => TeeVariant::DstackAwsNitroTpm, } } } @@ -1204,26 +1372,26 @@ mod compatibility_tests { use scale::Encode; #[test] - fn attestation_mode_scale_discriminants_preserve_existing_wire_values() { - assert_eq!(AttestationMode::DstackTdx.encode(), vec![0]); - assert_eq!(AttestationMode::DstackGcpTdx.encode(), vec![1]); - assert_eq!(AttestationMode::DstackNitroEnclave.encode(), vec![2]); - assert_eq!(AttestationMode::DstackAmdSevSnp.encode(), vec![3]); - assert_eq!(AttestationMode::DstackAwsNitroTpm.encode(), vec![4]); + fn tee_variant_scale_discriminants_preserve_existing_wire_values() { + assert_eq!(TeeVariant::DstackTdx.encode(), vec![0]); + assert_eq!(TeeVariant::DstackGcpTdx.encode(), vec![1]); + assert_eq!(TeeVariant::DstackNitroEnclave.encode(), vec![2]); + assert_eq!(TeeVariant::DstackAmdSevSnp.encode(), vec![3]); + assert_eq!(TeeVariant::DstackAwsNitroTpm.encode(), vec![4]); } #[test] - fn attestation_mode_deserializes_canonical_names() { - let parse = |value| serde_json::from_str::<AttestationMode>(value).unwrap(); - assert_eq!(parse(r#""dstack-tdx""#), AttestationMode::DstackTdx); - assert_eq!(parse(r#""dstack-gcp-tdx""#), AttestationMode::DstackGcpTdx); + fn tee_variant_deserializes_canonical_names() { + let parse = |value| serde_json::from_str::<TeeVariant>(value).unwrap(); + assert_eq!(parse(r#""dstack-tdx""#), TeeVariant::DstackTdx); + assert_eq!(parse(r#""dstack-gcp-tdx""#), TeeVariant::DstackGcpTdx); assert_eq!( parse(r#""dstack-amd-sev-snp""#), - AttestationMode::DstackAmdSevSnp + TeeVariant::DstackAmdSevSnp ); assert_eq!( parse(r#""dstack-nitro-enclave""#), - AttestationMode::DstackNitroEnclave + TeeVariant::DstackNitroEnclave ); } @@ -1261,18 +1429,18 @@ mod compatibility_tests { } #[test] - fn dstack_attestation_mode_prefers_tdx_when_both_tdx_and_tsm_exist() { + fn dstack_tee_variant_prefers_tdx_when_both_tdx_and_tsm_exist() { assert_eq!( - choose_dstack_attestation_mode(true, true).unwrap(), - AttestationMode::DstackTdx + choose_dstack_tee_variant(true, true).unwrap(), + TeeVariant::DstackTdx ); } #[test] - fn dstack_attestation_mode_uses_snp_when_only_snp_exists() { + fn dstack_tee_variant_uses_snp_when_only_snp_exists() { assert_eq!( - choose_dstack_attestation_mode(false, true).unwrap(), - AttestationMode::DstackAmdSevSnp + choose_dstack_tee_variant(false, true).unwrap(), + TeeVariant::DstackAmdSevSnp ); } } @@ -1339,6 +1507,16 @@ impl<T> Attestation<T> { self.tdx_quote().map(|q| q.quote.clone()) } + /// Populate `preimage` on every V2 runtime event in the TDX event log. + /// + /// Useful before serializing an attestation so relying parties get the + /// digest pre-images alongside events. + pub fn fill_event_preimages(&mut self) { + if let Some(q) = self.tdx_quote_mut() { + cc_eventlog::tdx::fill_v2_preimages(&mut q.event_log); + } + } + /// Get TDX event log bytes pub fn get_tdx_event_log_bytes(&self) -> Option<Vec<u8>> { self.tdx_quote() @@ -1347,35 +1525,23 @@ impl<T> Attestation<T> { /// Get TDX event log string with RTMR[0-2] payloads stripped to reduce size. /// Only digests are kept for boot-time events; runtime events (RTMR3) retain full payload. - pub fn get_tdx_event_log_string(&self) -> Option<String> { - self.get_tdx_event_log_string_for_config("") - } - - /// Get TDX event log string for a vm_config. /// - /// Always keeps the `ACPI DATA` marker payloads on the three RTMR0 ACPI - /// digest events, regardless of the vm_config's `tdx_attestation_variant`, - /// so callers that consume the top-level `event_log` can semantically - /// identify the ACPI table digest events without consulting the - /// versioned attestation field, and a verifier can choose lite - /// verification for any TDX boot rather than only ones resolved to lite - /// at launch. - /// - /// `config` is accepted for API stability but no longer changes the - /// result. - pub fn get_tdx_event_log_string_for_config(&self, _config: &str) -> Option<String> { + pub fn get_tdx_event_log_string(&self) -> Option<String> { self.tdx_quote().map(|q| { - let stripped: Vec<_> = q + let mut stripped: Vec<_> = q .event_log .iter() - .map(|e| { - let mut stripped = e.stripped(); - if is_tdx_acpi_data_event(e) { - stripped.event_payload = e.event_payload.clone(); + .map(|event| { + let mut stripped = event.stripped(); + // Keep the marker used by TDX-lite verification to identify + // the three RTMR0 ACPI digest events. + if cc_eventlog::tdx::is_tdx_acpi_data_event(event) { + stripped.event_payload = event.event_payload.clone(); } stripped }) .collect(); + cc_eventlog::tdx::fill_v2_preimages(&mut stripped); serde_json::to_string(&stripped).unwrap_or_default() }) } @@ -1449,7 +1615,7 @@ struct Mrs { fn key_provider_info_from_mr_config(mr_config: &MrConfigV3) -> Result<Vec<u8>> { serde_json::to_vec(&KeyProviderInfo::new( mr_config.key_provider_name().to_string(), - hex::encode(&mr_config.key_provider_id), + hex::encode(mr_config.key_provider_id.as_deref().unwrap_or_default()), )) .context("Failed to serialize key provider info") } @@ -1509,14 +1675,15 @@ fn decode_app_info_sev_snp( let mrs = decode_mr_sev_snp(&parsed.measurement, &parsed.host_data); Ok(AppInfo { - app_id: mr_config.app_id, - instance_id: mr_config.instance_id, + app_id: mr_config.app_id.unwrap_or_default(), + instance_id: mr_config.instance_id.unwrap_or_default(), device_id: sha256(parsed.chip_id).to_vec(), mr_system: mrs.mr_system, mr_aggregated: mrs.mr_aggregated, key_provider_info, os_image_hash, compose_hash: mr_config.compose_hash, + init_script_hashes: mr_config.init_script_hashes, }) } @@ -1626,15 +1793,15 @@ fn decode_mr_tdx_from_quote( } async fn verify_tdx_quote_with_events( - pccs_url: Option<&str>, + verifier: &AttestationVerifier, quote: &[u8], runtime_events: &[RuntimeEvent], report_data: &[u8; 64], ) -> Result<TdxVerifiedReport> { - let tdx_report = dcap_collateral_client(pccs_url)? - .fetch_and_verify(quote) + let tdx_report = verifier + .verify_tdx_quote(quote) .await - .context("Failed to get collateral")?; + .context("failed to verify TDX quote")?; validate_tcb(&tdx_report)?; let td_report = tdx_report.report.as_td10().context("no td report")?; @@ -1654,18 +1821,16 @@ async fn verify_tdx_quote_with_events( } fn verify_aws_nitro_tpm_attestation_doc( + verifier: &AttestationVerifier, attestation_doc: &[u8], runtime_events: &[RuntimeEvent], report_data: &[u8; 64], now: Option<SystemTime>, ) -> Result<AwsNitroTpmVerifiedReport> { - let verified_report = nsm_qvl::verify_attestation( - attestation_doc, - nsm_qvl::AWS_NITRO_ENCLAVES_ROOT_G1, - None, - now, - ) - .context("COSE attestation document verification failed")?; + let verified_report = verifier + .aws_nitro_tpm + .verify(attestation_doc, None, now) + .context("COSE attestation document verification failed")?; let Some(user_data) = verified_report.user_data.clone() else { bail!("NitroTPM attestation document does not contain user_data"); @@ -1841,6 +2006,10 @@ impl<T: GetDeviceId> Attestation<T> { key_provider_info, os_image_hash, compose_hash, + init_script_hashes: Some(find_event_payloads( + &self.runtime_events, + "init-script-hash", + )), } }; @@ -1931,6 +2100,12 @@ impl<T> Attestation<T> { self.find_event(event).map(|event| event.payload) } + /// SHA-256 payloads of all measured init scripts, in execution order. + /// Application-emitted events after `system-ready` are excluded. + pub fn decode_init_script_hashes(&self) -> Vec<Vec<u8>> { + find_event_payloads(&self.runtime_events, "init-script-hash") + } + fn find_event_hex_payload(&self, event: &str) -> Result<String> { self.find_event(event) .map(|event| hex::encode(&event.payload)) @@ -1997,50 +2172,68 @@ impl Attestation { } pub fn quote_with_app_id(report_data: &[u8; 64], app_id: Option<[u8; 20]>) -> Result<Self> { + Self::quote_with_app_id_and_sys_config(report_data, app_id, None) + } + + /// Create an attestation using an explicit sys-config path. + pub fn quote_with_sys_config( + report_data: &[u8; 64], + sys_config: &std::path::Path, + ) -> Result<Self> { + Self::quote_with_app_id_and_sys_config(report_data, None, Some(sys_config)) + } + + fn quote_with_app_id_and_sys_config( + report_data: &[u8; 64], + app_id: Option<[u8; 20]>, + sys_config: Option<&std::path::Path>, + ) -> Result<Self> { // Lock to prevent concurrent quote generation (TDX driver doesn't support it) let _guard = QUOTE_LOCK .lock() .map_err(|_| anyhow!("Quote lock poisoned"))?; - let mode = AttestationMode::detect()?; + let mode = detect_tee_variant()?; let config = match mode { - AttestationMode::DstackAmdSevSnp - | AttestationMode::DstackTdx - | AttestationMode::DstackGcpTdx - // AWS: prefer host-shared sys-config vm_config (carries - // aws_measurement + unified os_image_hash); validated below. - | AttestationMode::DstackAwsNitroTpm => { - read_vm_config().context("Failed to read vm config")? + TeeVariant::DstackAmdSevSnp + | TeeVariant::DstackTdx + | TeeVariant::DstackGcpTdx + // AWS prefers host-shared vm_config because it carries the + // aws_measurement and unified os_image_hash validated below. + | TeeVariant::DstackAwsNitroTpm => { + read_vm_config(sys_config).context("Failed to read vm config")? } - // NitroEnclave derives config from the quote's image hash below. - AttestationMode::DstackNitroEnclave => String::new(), + // NitroEnclave derives config from the signed image hash below. + TeeVariant::DstackNitroEnclave => String::new(), }; let runtime_events = match mode { - AttestationMode::DstackTdx - | AttestationMode::DstackGcpTdx - | AttestationMode::DstackAwsNitroTpm => { + TeeVariant::DstackTdx | TeeVariant::DstackGcpTdx | TeeVariant::DstackAwsNitroTpm => { RuntimeEvent::read_all().context("Failed to read runtime events")? } - AttestationMode::DstackAmdSevSnp => vec![], - AttestationMode::DstackNitroEnclave => match app_id { - Some(app_id) => vec![RuntimeEvent::new("app-id".to_string(), app_id.to_vec())], + TeeVariant::DstackAmdSevSnp => vec![], + TeeVariant::DstackNitroEnclave => match app_id { + Some(app_id) => vec![RuntimeEvent::new( + "app-id".to_string(), + app_id.to_vec(), + EventLogVersion::V1, + )], None => vec![], }, }; let mut quote = match mode { - AttestationMode::DstackTdx => { + TeeVariant::DstackTdx => { let quote = tdx_attest::get_quote(report_data).context("Failed to get quote")?; let event_log = cc_eventlog::tdx::read_event_log().context("Failed to read event log")?; AttestationQuote::DstackTdx(TdxQuote { quote, event_log }) } - AttestationMode::DstackAmdSevSnp => { + TeeVariant::DstackAmdSevSnp => { let quote = crate::sev_snp::get_report(*report_data) .context("Failed to get SEV-SNP report")?; AttestationQuote::DstackAmdSevSnp(quote) } - AttestationMode::DstackGcpTdx => { + TeeVariant::DstackGcpTdx => { let quote = tdx_attest::get_quote(report_data).context("Failed to get quote")?; let event_log = cc_eventlog::tdx::read_event_log().context("Failed to read event log")?; @@ -2056,12 +2249,12 @@ impl Attestation { tpm_quote, }) } - AttestationMode::DstackNitroEnclave => { + TeeVariant::DstackNitroEnclave => { let nsm_quote = nsm_attest::get_attestation(report_data) .context("Failed to get NSM attestation")?; AttestationQuote::DstackNitroEnclave(DstackNitroQuote { nsm_quote }) } - AttestationMode::DstackAwsNitroTpm => { + TeeVariant::DstackAwsNitroTpm => { // Challenge binding is report_data → NitroTPM user_data only // (same role as TDX/GCP report_data; no separate nonce/public_key). let attestation_doc = crate::aws_nitro_tpm::attestation_document(report_data) @@ -2118,7 +2311,7 @@ impl Attestation { }; if let AttestationQuote::DstackAmdSevSnp(quote) = &mut quote { quote.mr_config = - read_mr_config_document()?.context("amd sev-snp mr_config is missing")?; + read_mr_config_document(sys_config)?.context("amd sev-snp mr_config is missing")?; } Ok(Self { @@ -2136,56 +2329,37 @@ impl Attestation { self.into() } - /// Verify the quote with optional custom time (testing hook) - pub async fn verify_with_time( - self, - pccs_url: Option<&str>, - now: Option<SystemTime>, - ) -> Result<VerifiedAttestation> { - self.verify_with_time_with_amd_kds_client(pccs_url, now, None) - .await - } - - /// Verify the quote with a caller-owned AMD KDS client. - pub async fn verify_with_amd_kds_client( - self, - pccs_url: Option<&str>, - amd_kds_client: &AmdKdsClient, - ) -> Result<VerifiedAttestation> { - self.verify_with_time_with_amd_kds_client(pccs_url, None, Some(amd_kds_client)) - .await + pub async fn verify(self, verifier: &AttestationVerifier) -> Result<VerifiedAttestation> { + self.verify_with_time(verifier, None).await } - async fn verify_with_time_with_amd_kds_client( + pub async fn verify_with_time( self, - pccs_url: Option<&str>, + verifier: &AttestationVerifier, now: Option<SystemTime>, - amd_kds_client: Option<&AmdKdsClient>, ) -> Result<VerifiedAttestation> { let report = match &self.quote { AttestationQuote::DstackTdx(q) => { - let report = self.verify_tdx(pccs_url, &q.quote).await?; + let report = self.verify_tdx(verifier, &q.quote).await?; DstackVerifiedReport::DstackTdx(report) } AttestationQuote::DstackAmdSevSnp(q) => { - let owned_kds_client; - let kds_client = match amd_kds_client { - Some(client) => client, - None => { - owned_kds_client = AmdKdsClient::new()?; - &owned_kds_client - } - }; - let verified = kds_client - .verify_evidence_with_kds_fallback(&q.report, &q.cert_chain, &self.report_data) + let verified = verifier + .sev_snp + .fetch_and_verify( + &verifier.amd_kds, + &q.report, + &q.cert_chain, + &self.report_data, + ) .await?; verify_snp_mr_config_host_data(&q.mr_config, &verified.host_data)?; DstackVerifiedReport::DstackAmdSevSnp(verified) } AttestationQuote::DstackGcpTdx(q) => { - let tdx_report = self.verify_tdx(pccs_url, &q.tdx_quote.quote).await?; + let tdx_report = self.verify_tdx(verifier, &q.tdx_quote.quote).await?; let tpm_report = self - .verify_tpm(&q.tpm_quote, &sha256(&q.tdx_quote.quote)) + .verify_tpm(verifier, &q.tpm_quote, &sha256(&q.tdx_quote.quote)) .await .context("Failed to verify TPM quote")?; DstackVerifiedReport::DstackGcpTdx { @@ -2195,13 +2369,14 @@ impl Attestation { } AttestationQuote::DstackNitroEnclave(quote) => { let report = self - .verify_nitro_enclave_with_time(quote, now) + .verify_nitro_enclave_with_time(verifier, quote, now) .await .context("Failed to verify Nitro Enclave")?; DstackVerifiedReport::DstackNitroEnclave(report) } AttestationQuote::DstackAwsNitroTpm(quote) => { let report = verify_aws_nitro_tpm_attestation_doc( + verifier, "e.attestation_doc, &self.runtime_events, &self.report_data, @@ -2212,6 +2387,18 @@ impl Attestation { } }; + match &self.quote { + AttestationQuote::DstackTdx(q) => { + cc_eventlog::tdx::validate_v2_preimages(&q.event_log) + .context("Failed to validate TDX V2 event digest preimages")?; + } + AttestationQuote::DstackGcpTdx(q) => { + cc_eventlog::tdx::validate_v2_preimages(&q.tdx_quote.event_log) + .context("Failed to validate TDX V2 event digest preimages")?; + } + _ => {} + } + Ok(VerifiedAttestation { quote: self.quote, runtime_events: self.runtime_events, @@ -2221,27 +2408,41 @@ impl Attestation { }) } - /// Wrap into a versioned attestation for encoding - pub fn into_versioned(self) -> VersionedAttestation { - VersionedAttestation::V0 { attestation: self } + /// Wrap into a versioned attestation for encoding. + /// + /// When any runtime event uses a non-V1 event-log version, force the V1 + /// msgpack wire format so the `version` field is preserved (SCALE + /// V0 skips it for legacy binary compat). Otherwise default to V0 for + /// backward compat with callers that expect the SCALE format. + pub fn into_versioned(mut self) -> VersionedAttestation { + // V2 event digests cannot be reconstructed from the serialized event + // fields alone. Populate their canonical preimages before the legacy + // quote is projected into either wire schema. + self.fill_event_preimages(); + let has_v2 = self + .runtime_events + .iter() + .any(|e| !matches!(e.version, EventLogVersion::V1)); + if has_v2 { + VersionedAttestation::V1 { + attestation: self.into(), + } + } else { + VersionedAttestation::V0 { attestation: self } + } } /// Verify the quote pub async fn verify_with_ra_pubkey( self, ra_pubkey_der: &[u8], - pccs_url: Option<&str>, + verifier: &AttestationVerifier, ) -> Result<VerifiedAttestation> { let expected_report_data = QuoteContentType::RaTlsCert.to_report_data(ra_pubkey_der); if self.report_data != expected_report_data { bail!("report data mismatch"); } - self.verify(pccs_url).await - } - - /// Verify the quote - pub async fn verify(self, pccs_url: Option<&str>) -> Result<VerifiedAttestation> { - self.verify_with_time(pccs_url, None).await + self.verify(verifier).await } /// Verify Nitro Enclave attestation with optional custom time (testing hook) @@ -2252,18 +2453,16 @@ impl Attestation { /// 3. Validates user_data matches expected report_data async fn verify_nitro_enclave_with_time( &self, + verifier: &AttestationVerifier, nsm_quote: &DstackNitroQuote, now: Option<SystemTime>, ) -> Result<NitroVerifiedReport> { // Verify COSE signature and certificate chain using nsm-qvl // CRL fetch is unreliable (e.g. 403 from S3), so keep it disabled here by default. - let verified_report = nsm_qvl::verify_attestation( - &nsm_quote.nsm_quote, - nsm_qvl::AWS_NITRO_ENCLAVES_ROOT_G1, - None, - now, - ) - .context("NSM attestation verification failed")?; + let verified_report = verifier + .aws_nitro_enclave + .verify(&nsm_quote.nsm_quote, None, now) + .context("NSM attestation verification failed")?; // Verify user_data matches report_data let Some(user_data) = verified_report.user_data.clone() else { @@ -2288,13 +2487,14 @@ impl Attestation { async fn verify_tpm( &self, + verifier: &AttestationVerifier, quote: &TpmQuote, qualifying_data: &[u8], ) -> Result<TpmVerifiedReport> { - let report = tpm_qvl::get_collateral_and_verify(quote).await?; + let report = verifier.gcp_tpm.fetch_and_verify(quote).await?; let pcr_ind = self .quote - .mode() + .variant() .tpm_event_pcr_and_bank() .map(|(pcr, _)| pcr) .context("Failed to get event PCR no")?; @@ -2315,11 +2515,15 @@ impl Attestation { Ok(report) } - async fn verify_tdx(&self, pccs_url: Option<&str>, quote: &[u8]) -> Result<TdxVerifiedReport> { - let tdx_report = dcap_collateral_client(pccs_url)? - .fetch_and_verify(quote) + async fn verify_tdx( + &self, + verifier: &AttestationVerifier, + quote: &[u8], + ) -> Result<TdxVerifiedReport> { + let tdx_report = verifier + .verify_tdx_quote(quote) .await - .context("Failed to get collateral")?; + .context("failed to verify TDX quote")?; validate_tcb(&tdx_report)?; let td_report = tdx_report.report.as_td10().context("no td report")?; @@ -2398,12 +2602,156 @@ pub struct AppInfo { /// Key provider info #[serde(with = "hex_bytes")] pub key_provider_info: Vec<u8>, + /// Optional SHA-256 pins for init scripts, in execution order. `None` + /// means the evidence did not bind this field. On SEV-SNP, `Some(vec![])` + /// explicitly binds an empty script list. On TDX and Nitro it only means + /// that no `init-script-hash` events were measured before `system-ready`; + /// pre-0.6.0 images emit no such events even when they run an init script. + #[serde(default, with = "dstack_types::init_script_hashes::option")] + pub init_script_hashes: Option<Vec<Vec<u8>>>, } #[cfg(test)] mod tests { use super::*; + #[test] + fn app_info_defaults_missing_init_script_hashes() { + let app_info: AppInfo = serde_json::from_value(serde_json::json!({ + "app_id": "", + "compose_hash": "", + "instance_id": "", + "device_id": "", + "mr_system": "0000000000000000000000000000000000000000000000000000000000000000", + "mr_aggregated": "0000000000000000000000000000000000000000000000000000000000000000", + "os_image_hash": "", + "key_provider_info": "" + })) + .unwrap(); + assert!(app_info.init_script_hashes.is_none()); + } + + #[test] + fn app_info_preserves_explicit_empty_init_script_hashes() { + let app_info: AppInfo = serde_json::from_value(serde_json::json!({ + "app_id": "", + "compose_hash": "", + "instance_id": "", + "device_id": "", + "mr_system": "0000000000000000000000000000000000000000000000000000000000000000", + "mr_aggregated": "0000000000000000000000000000000000000000000000000000000000000000", + "os_image_hash": "", + "key_provider_info": "", + "init_script_hashes": [] + })) + .unwrap(); + assert_eq!(app_info.init_script_hashes, Some(Vec::new())); + } + + #[test] + fn external_trust_anchor_requires_explicit_insecure_opt_in() { + let config = AttestationVerifierConfig { + root_ca: RootCaPaths { + tdx: Some("/tmp/mock-tdx-root.pem".into()), + ..Default::default() + }, + ..Default::default() + }; + let error = AttestationVerifier::load(&config) + .err() + .unwrap() + .to_string(); + assert!(error.contains("insecure_allow_external_trust_anchors is false")); + } + + #[test] + fn production_attestation_verifier_loads_all_safe_defaults() { + AttestationVerifier::load(&AttestationVerifierConfig::default()) + .expect("production roots and URLs must load"); + } + + #[test] + fn opted_in_external_root_is_read_during_load() { + let config = AttestationVerifierConfig { + insecure_allow_external_trust_anchors: true, + root_ca: RootCaPaths { + tdx: Some("/definitely/missing/tdx-root.pem".into()), + ..Default::default() + }, + ..Default::default() + }; + let error = AttestationVerifier::load(&config) + .err() + .unwrap() + .to_string(); + assert!(error.contains("failed to read TDX root CA")); + } + + #[test] + fn root_file_is_loaded_without_environment_indirection() { + let path = std::env::temp_dir().join(format!("dstack-root-ca-test-{}", std::process::id())); + fs_err::write(&path, b"test root").unwrap(); + assert_eq!( + read_root_file(Some(&path), "test").unwrap(), + Some(b"test root".to_vec()) + ); + fs_err::remove_file(path).unwrap(); + } + + #[test] + fn every_external_root_is_parsed_during_load() { + let path = std::env::temp_dir().join(format!( + "dstack-invalid-root-ca-test-{}", + std::process::id() + )); + fs_err::write(&path, b"not a certificate").unwrap(); + let roots = [ + RootCaPaths { + tdx: Some(path.clone()), + ..Default::default() + }, + RootCaPaths { + gcp_tpm: Some(path.clone()), + ..Default::default() + }, + RootCaPaths { + aws_nitro_enclave: Some(path.clone()), + ..Default::default() + }, + RootCaPaths { + aws_nitro_tpm: Some(path.clone()), + ..Default::default() + }, + RootCaPaths { + sev_snp_milan: Some(path.clone()), + ..Default::default() + }, + RootCaPaths { + sev_snp_genoa: Some(path.clone()), + ..Default::default() + }, + RootCaPaths { + sev_snp_turin: Some(path.clone()), + ..Default::default() + }, + ]; + for root_ca in roots { + let config = AttestationVerifierConfig { + insecure_allow_external_trust_anchors: true, + root_ca, + ..Default::default() + }; + let error = AttestationVerifier::load(&config) + .err() + .expect("invalid external root must fail during load"); + assert!( + format!("{error:#}").contains("root CA"), + "unexpected error: {error:#}" + ); + } + fs_err::remove_file(path).unwrap(); + } + fn patch_v1_report_data(attestation: AttestationV1, report_data: [u8; 64]) -> AttestationV1 { attestation.with_report_data(report_data) } @@ -2428,42 +2776,63 @@ mod tests { digest: vec![event_type as u8; 48], event: String::new(), event_payload: event_payload.to_vec(), + version: EventLogVersion::V1, + preimage: None, } } #[test] - fn tdx_event_log_string_always_keeps_acpi_data_payloads() { + fn get_quote_event_log_keeps_acpi_data_payloads() { let mut attestation = dummy_tdx_attestation([0u8; 64]); let AttestationQuote::DstackTdx(tdx_quote) = &mut attestation.quote else { panic!("expected TDX attestation"); }; tdx_quote.event_log = vec![ + tdx_event(0, 10, b"ACPI DATA"), + tdx_event(0, 10, b"ACPI DATA"), tdx_event(0, 10, b"ACPI DATA"), tdx_event(0, 4, b"boot-payload"), - tdx_event(3, 8, b"runtime-payload"), + tdx_event( + 3, + cc_eventlog::DSTACK_RUNTIME_EVENT_TYPE, + b"v1-runtime-payload", + ), + { + let mut event = tdx_event( + 3, + cc_eventlog::DSTACK_RUNTIME_EVENT_TYPE, + b"v2-runtime-payload", + ); + event.version = EventLogVersion::V2; + event + }, ]; // The ACPI DATA marker payload is retained regardless of the // vm_config's tdx_attestation_variant (including no vm_config at // all), so a verifier can choose lite verification for any TDX boot. - for config in [ - r#"{"tdx_attestation_variant":"lite"}"#, - r#"{"tdx_attestation_variant":"legacy"}"#, - "", - ] { - let events: Vec<TdxEvent> = serde_json::from_str( - &attestation - .get_tdx_event_log_string_for_config(config) - .expect("TDX event log"), - ) - .unwrap_or_else(|e| panic!("decode event log for config {config:?}: {e}")); - assert_eq!( - events[0].event_payload, b"ACPI DATA", - "config {config:?} must keep the ACPI DATA marker payload" - ); - assert!(events[1].event_payload.is_empty()); - assert!(events[2].event_payload.is_empty()); - } + let events: Vec<TdxEvent> = serde_json::from_str( + &attestation + .get_tdx_event_log_string() + .expect("TDX event log"), + ) + .unwrap_or_else(|e| panic!("decode GetQuote event log: {e}")); + assert_eq!( + events + .iter() + .filter(|event| cc_eventlog::tdx::is_tdx_acpi_data_event(event)) + .count(), + 3, + "GetQuote must retain all three TDX-lite ACPI DATA markers" + ); + assert!(events[3].event_payload.is_empty()); + assert_eq!(events[4].event_payload, b"v1-runtime-payload"); + assert!( + events[4].preimage.is_none(), + "V1 output must remain unchanged" + ); + assert_eq!(events[5].event_payload, b"v2-runtime-payload"); + assert!(events[5].preimage.is_some(), "V2 must include its preimage"); } #[test] @@ -2472,7 +2841,10 @@ mod tests { let content = b"test content"; let report_data = content_type.to_report_data(content); - assert_eq!(hex::encode(report_data), "7ea0b744ed5e9c0c83ff9f575668e1697652cd349f2027cdf26f918d4c53e8cd50b5ea9b449b4c3d50e20ae00ec29688d5a214e8daff8a10041f5d624dae8a01"); + assert_eq!( + hex::encode(report_data), + "7ea0b744ed5e9c0c83ff9f575668e1697652cd349f2027cdf26f918d4c53e8cd50b5ea9b449b4c3d50e20ae00ec29688d5a214e8daff8a10041f5d624dae8a01" + ); // Test SHA-256 let result = content_type @@ -2552,6 +2924,40 @@ mod tests { assert!(matches!(upgraded.stack, StackEvidence::Dstack { .. })); } + #[test] + fn v1_dstack_with_v1_events_converts_losslessly_to_legacy() { + let mut legacy = dummy_tdx_attestation([0x5a; 64]); + legacy.runtime_events.push(cc_eventlog::RuntimeEvent::new( + "legacy-event".into(), + vec![1, 2, 3], + cc_eventlog::EventLogVersion::V1, + )); + let converted = legacy.clone().into_v1().try_into_legacy().unwrap(); + assert_eq!(converted.report_data, legacy.report_data); + assert_eq!(converted.runtime_events.len(), 1); + assert!(matches!( + converted.into_versioned(), + VersionedAttestation::V0 { .. } + )); + } + + #[test] + fn v1_conversion_rejects_lossy_legacy_projection() { + let pod = dummy_tdx_attestation([0x5b; 64]) + .into_v1() + .into_dstack_pod("payload".into()); + assert!(pod.try_into_legacy().is_err()); + let mut v2 = dummy_tdx_attestation([0x5c; 64]).into_v1(); + if let StackEvidence::Dstack { runtime_events, .. } = &mut v2.stack { + runtime_events.push(cc_eventlog::RuntimeEvent::new( + "v2-event".into(), + vec![4, 5, 6], + cc_eventlog::EventLogVersion::V2, + )); + } + assert!(v2.try_into_legacy().is_err()); + } + #[test] fn versioned_v0_projects_to_v1() { let projected = dummy_tdx_attestation([5u8; 64]).into_versioned().into_v1(); @@ -2570,6 +2976,81 @@ mod tests { } } + #[test] + fn into_versioned_uses_v0_when_all_events_are_v1() { + let mut att = dummy_tdx_attestation([7u8; 64]); + att.runtime_events.push(cc_eventlog::RuntimeEvent::new( + "app-id".into(), + vec![1, 2, 3], + cc_eventlog::EventLogVersion::V1, + )); + let versioned = att.into_versioned(); + assert!( + matches!(versioned, VersionedAttestation::V0 { .. }), + "V1-only events should stay on the V0/SCALE wire format" + ); + } + + #[test] + fn into_versioned_upgrades_to_v1_when_any_event_is_v2() { + let mut att = dummy_tdx_attestation([8u8; 64]); + let AttestationQuote::DstackTdx(tdx_quote) = &mut att.quote else { + panic!("expected TDX attestation"); + }; + tdx_quote.event_log.push( + cc_eventlog::RuntimeEvent::new( + "compose-hash".into(), + vec![4, 5, 6], + cc_eventlog::EventLogVersion::V2, + ) + .into(), + ); + att.runtime_events.push(cc_eventlog::RuntimeEvent::new( + "app-id".into(), + vec![1, 2, 3], + cc_eventlog::EventLogVersion::V1, + )); + att.runtime_events.push(cc_eventlog::RuntimeEvent::new( + "compose-hash".into(), + vec![4, 5, 6], + cc_eventlog::EventLogVersion::V2, + )); + // RA-TLS certificates use the stripped representation. Its runtime + // events must retain the advertised digest paired with each preimage. + let encoded = att.into_versioned().into_stripped().to_bytes().unwrap(); + let VersionedAttestation::V1 { attestation } = + VersionedAttestation::from_bytes(&encoded).unwrap() + else { + panic!("presence of a V2 event must force the V1 msgpack wire format"); + }; + let PlatformEvidence::Tdx { event_log, .. } = attestation.platform else { + panic!("expected TDX platform evidence"); + }; + assert!( + event_log[0].preimage.is_some(), + "serialized V2 TDX events must carry their canonical digest preimage" + ); + cc_eventlog::tdx::validate_v2_preimages(&event_log).unwrap(); + } + fn v1_event(event: String, payload: Vec<u8>) -> RuntimeEvent { + RuntimeEvent::new(event, payload, EventLogVersion::V1) + } + + #[test] + fn init_script_hashes_exclude_application_events_after_system_ready() { + let events = vec![ + v1_event("init-script-hash".into(), vec![0x11; 32]), + v1_event("init-script-hash".into(), vec![0x22; 32]), + v1_event("system-ready".into(), Vec::new()), + v1_event("init-script-hash".into(), vec![0xff; 32]), + ]; + + assert_eq!( + find_event_payloads(&events, "init-script-hash"), + vec![vec![0x11; 32], vec![0x22; 32]] + ); + } + #[test] fn nitro_pcrs_from_verified_extracts_0_1_2() { let mut map = std::collections::BTreeMap::new(); @@ -2621,13 +3102,13 @@ mod tests { let mr_key_provider = sha256(b"aws nitrotpm key provider"); let events = vec![ - RuntimeEvent::new("system-preparing".into(), Vec::new()), - RuntimeEvent::new("app-id".into(), vec![0x11; 20]), - RuntimeEvent::new("compose-hash".into(), vec![0x22; 32]), - RuntimeEvent::new("instance-id".into(), vec![0x33; 20]), - RuntimeEvent::new("boot-mr-done".into(), Vec::new()), - RuntimeEvent::new("key-provider".into(), b"tpm".to_vec()), - RuntimeEvent::new("system-ready".into(), Vec::new()), + v1_event("system-preparing".into(), Vec::new()), + v1_event("app-id".into(), vec![0x11; 20]), + v1_event("compose-hash".into(), vec![0x22; 32]), + v1_event("instance-id".into(), vec![0x33; 20]), + v1_event("boot-mr-done".into(), Vec::new()), + v1_event("key-provider".into(), b"tpm".to_vec()), + v1_event("system-ready".into(), Vec::new()), ]; let replayed_pcr14 = cc_eventlog::replay_events::<Sha384>(&events, None); pcrs.insert(AWS_NITRO_TPM_EVENT_PCR, replayed_pcr14.to_vec()); @@ -2654,7 +3135,7 @@ mod tests { ); let mut changed_events = events.clone(); - changed_events[2] = RuntimeEvent::new("compose-hash".into(), vec![0xee; 32]); + changed_events[2] = v1_event("compose-hash".into(), vec![0xee; 32]); let changed_pcr14 = cc_eventlog::replay_events::<Sha384>(&changed_events, None); let mut changed_pcrs = pcrs.clone(); changed_pcrs.insert(AWS_NITRO_TPM_EVENT_PCR, changed_pcr14.to_vec()); @@ -2708,14 +3189,14 @@ mod tests { // Single PCR14 lane: all events (including after system-ready) are // measured and must be replayed for full (non-boottime) decode. let events = vec![ - RuntimeEvent::new("system-preparing".into(), Vec::new()), - RuntimeEvent::new("app-id".into(), vec![0x11; 20]), - RuntimeEvent::new("compose-hash".into(), vec![0x22; 32]), - RuntimeEvent::new("instance-id".into(), vec![0x33; 20]), - RuntimeEvent::new("boot-mr-done".into(), Vec::new()), - RuntimeEvent::new("storage-fs".into(), b"ext4".to_vec()), - RuntimeEvent::new("system-ready".into(), Vec::new()), - RuntimeEvent::new("app-runtime".into(), b"ready".to_vec()), + v1_event("system-preparing".into(), Vec::new()), + v1_event("app-id".into(), vec![0x11; 20]), + v1_event("compose-hash".into(), vec![0x22; 32]), + v1_event("instance-id".into(), vec![0x33; 20]), + v1_event("boot-mr-done".into(), Vec::new()), + v1_event("storage-fs".into(), b"ext4".to_vec()), + v1_event("system-ready".into(), Vec::new()), + v1_event("app-runtime".into(), b"ready".to_vec()), ]; let full_pcr = cc_eventlog::replay_events::<Sha384>(&events, None); @@ -2763,10 +3244,7 @@ mod tests { .iter() .take_while(|event| event.event != "boot-mr-done") .cloned() - .chain(std::iter::once(RuntimeEvent::new( - "boot-mr-done".into(), - Vec::new(), - ))) + .chain(std::iter::once(v1_event("boot-mr-done".into(), Vec::new()))) .collect(); let mut early_pcrs = pcrs.clone(); early_pcrs.insert(AWS_NITRO_TPM_EVENT_PCR, early_pcr.to_vec()); @@ -2783,4 +3261,38 @@ mod tests { ); Ok(()) } + + #[test] + fn versioned_wire_formats_reject_malformed_boundaries() { + assert!(VersionedAttestation::from_bytes(&[]).is_err()); + assert!(VersionedAttestation::from_bytes(&[0xff]).is_err()); + assert!(VersionedAttestation::from_bytes(&vec![0xff; MAX_ATTESTATION_BYTES + 1]).is_err()); + + let legacy = dummy_tdx_attestation([0x31; 64]).into_versioned(); + assert!(matches!(legacy, VersionedAttestation::V0 { .. })); + let legacy_bytes = legacy.to_bytes().unwrap(); + let decoded = VersionedAttestation::from_bytes(&legacy_bytes).unwrap(); + assert_eq!(decoded.into_v1().report_data().unwrap(), [0x31; 64]); + assert!(VersionedAttestation::from_bytes(&legacy_bytes[..legacy_bytes.len() - 1]).is_err()); + assert!( + VersionedAttestation::from_bytes(&[legacy_bytes.as_slice(), &[0xaa]].concat()).is_err() + ); + + let current = dummy_tdx_attestation([0x32; 64]) + .into_v1() + .into_dstack_pod("versioned-boundary".into()); + let current = VersionedAttestation::V1 { + attestation: current, + }; + let current_bytes = current.to_bytes().unwrap(); + let decoded = VersionedAttestation::from_bytes(¤t_bytes).unwrap(); + assert_eq!(decoded.into_v1().report_data().unwrap(), [0x32; 64]); + assert!( + VersionedAttestation::from_bytes(¤t_bytes[..current_bytes.len() - 1]).is_err() + ); + assert!( + VersionedAttestation::from_bytes(&[current_bytes.as_slice(), &[0xaa]].concat()) + .is_err() + ); + } } diff --git a/dstack/dstack-attest/src/lib.rs b/dstack/dstack-attest/src/lib.rs index a75adc811..dd088c1df 100644 --- a/dstack/dstack-attest/src/lib.rs +++ b/dstack/dstack-attest/src/lib.rs @@ -2,16 +2,14 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::sync::{LazyLock, Mutex}; - use anyhow::Context; -use cc_eventlog::RuntimeEvent; +use cc_eventlog::{EventLogVersion, RuntimeEvent}; pub use cc_eventlog as ccel; pub use tdx_attest as tdx; pub use tpm_attest as tpm; -use crate::attestation::AttestationMode; +use crate::attestation::{detect_tee_variant, TeeVariant}; pub mod amd_sev_snp; pub mod attestation; @@ -19,52 +17,141 @@ pub mod attestation; mod aws_nitro_tpm; #[cfg(feature = "quote")] mod sev_snp; +pub mod trust_anchors; mod v1; -/// Serializes measured event emission within this process. +const RUNTIME_EVENT_DIR: &str = "/run/log/dstack"; +const RUNTIME_EVENT_VERSION_FILE: &str = "/run/log/dstack/runtime_event_version"; +const RUNTIME_EVENT_LOCK_FILE: &str = "/run/log/dstack/runtime_event.lock"; + +/// Build the verifier a guest authenticates the KMS and the gateway with. /// -/// Appending to the event log and extending the platform measurement register -/// must happen atomically as a unit: the log order has to match the extension -/// order, otherwise replay during quote verification will not reproduce the -/// measured value. Concurrent callers, for example multiple `emit_event` RPCs -/// hitting the guest-agent at once, would otherwise be able to interleave their -/// log writes and register extensions. -static EMIT_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(())); - -/// Emit a dstack measured event and log the event. +/// Trust anchors are taken from [`trust_anchors::ANCHOR_DIR`] when that +/// directory holds a set published inside this guest. When it does not — the +/// only outcome on a production image — the vendor production roots apply. /// -/// Semantics match bare TDX RTMR3: every dstack event extends a single -/// append-only measurement register. +/// `collateral_urls` selects where signed collateral is fetched from; the trust +/// anchor still has to sign it. +pub fn default_verifier( + collateral_urls: &attestation::CollateralUrls, +) -> anyhow::Result<attestation::AttestationVerifier> { + use attestation::{AttestationVerifier, AttestationVerifierConfig}; + + let Some(root_ca) = + trust_anchors::load_anchors(std::path::Path::new(trust_anchors::ANCHOR_DIR)) + .context("failed to load local attestation anchors")? + else { + return AttestationVerifier::new_prod(Some(collateral_urls)); + }; + tracing::warn!( + dir = trust_anchors::ANCHOR_DIR, + "verifying attestation against external trust anchors published by the in-guest TEE \ + simulator; this guest cannot verify production evidence" + ); + AttestationVerifier::load(&AttestationVerifierConfig { + // The opt-in exists to make an operator acknowledge a non-production + // root in a hand-written service config. Nothing here is hand-written: + // the roots came from a guest-local directory `load_anchors` already + // authenticated, so the flag has no one left to warn. + insecure_allow_external_trust_anchors: true, + urls: collateral_urls.clone(), + root_ca, + }) +} + +/// Acquire the system-wide runtime event lock, blocking until it is available. /// -/// - TDX-family: RTMR3 -/// - GCP TPM: SHA256 PCR14 -/// - AWS NitroTPM: SHA384 PCR14 (not PCR23; no launch/runtime PCR split) -pub fn emit_runtime_event(event: &str, payload: &[u8]) -> anyhow::Result<()> { - let mode = match AttestationMode::detect() { - Ok(mode) => mode, - Err(err) => { - // DEV-ONLY fallback: no TEE hardware is present (e.g. a local - // no_tee dev CVM on a plain laptop with no TDX/SEV-SNP device). - // Skip the event log / RTMR extension instead of failing the - // boot. Real TEE hosts always resolve an AttestationMode above - // and take the unchanged path below; production attestation is - // enforced at a separate layer (KMS/auth-server reject any VM - // that cannot produce a valid quote), so this does not weaken - // production security. - tracing::warn!( - "no TEE attestation mode available ({err:#}); skipping runtime event '{event}' (no_tee dev mode)" +/// The wait is deliberately unbounded. The lock serializes the event-log append +/// with the measurement-register extension, and that ordering is what makes +/// replay reproduce the quoted register value. Proceeding after a timeout would +/// break the invariant, and failing after one would abort a boot that is merely +/// slow, so waiting is the only safe option. A holder that dies releases the +/// lock automatically (flock is dropped when the file descriptor closes), which +/// leaves a live but wedged holder as the sole way to block emission. +fn runtime_event_lock() -> anyhow::Result<fs_err::File> { + fs_err::create_dir_all(RUNTIME_EVENT_DIR) + .context("failed to create runtime event log directory")?; + let lock = fs_err::OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(RUNTIME_EVENT_LOCK_FILE) + .context("failed to open runtime event lock")?; + rustix::fs::flock(&lock, rustix::fs::FlockOperation::LockExclusive) + .context("failed to lock runtime event emission")?; + Ok(lock) +} + +/// Configure the system-wide digest format used by subsequently emitted events. +/// +/// The setting is persisted under `/run/log/dstack`, so separate dstack-util +/// processes share it. This must be called before [`emit_runtime_event`]. +/// Repeating the same configuration is allowed; changing it is rejected. +pub fn set_runtime_event_version(version: EventLogVersion) -> anyhow::Result<()> { + let _lock = runtime_event_lock()?; + set_runtime_event_version_file(RUNTIME_EVENT_VERSION_FILE, version) +} + +fn set_runtime_event_version_file( + path: impl AsRef<std::path::Path>, + version: EventLogVersion, +) -> anyhow::Result<()> { + let path = path.as_ref(); + let value = match version { + EventLogVersion::V1 => "1", + EventLogVersion::V2 => "2", + }; + match fs_err::read_to_string(path) { + Ok(configured) => { + anyhow::ensure!( + configured.trim() == value, + "runtime event version is already set to {} for this boot and cannot be \ + changed to {value}; the setting is fixed when system setup runs, so restart \ + the CVM to apply a new app-compose `event_log_version`", + configured.trim() ); - return Ok(()); + Ok(()) } - }; + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + safe_write::safe_write(path, value.as_bytes()) + .context("failed to write runtime event version") + } + Err(err) => Err(err).context("failed to read runtime event version"), + } +} - let event = RuntimeEvent::new(event.to_string(), payload.to_vec()); +fn runtime_event_version() -> anyhow::Result<EventLogVersion> { + runtime_event_version_file(RUNTIME_EVENT_VERSION_FILE) +} - // Hold the lock across both the log append and the register extension so - // that the on-disk log order always matches the RTMR extension order. - let _guard = EMIT_LOCK.lock().unwrap_or_else(|e| e.into_inner()); +fn runtime_event_version_file( + path: impl AsRef<std::path::Path>, +) -> anyhow::Result<EventLogVersion> { + let value = fs_err::read_to_string(path).context( + "runtime event version is not configured; complete dstack system setup before emitting events", + )?; + match value.trim() { + "1" => Ok(EventLogVersion::V1), + "2" => Ok(EventLogVersion::V2), + value => anyhow::bail!("invalid runtime event version: {value}"), + } +} - event.emit().context("Failed to emit runtime event")?; +/// Emit a dstack measured event using the system-configured digest format. +/// +/// The event-log append and platform-register extension are serialized by a +/// system-wide file lock so their ordering cannot diverge across processes. +/// +/// - TDX-family: RTMR3 +/// - GCP TPM: SHA256 PCR14 +/// - AWS NitroTPM: SHA384 PCR14 +pub fn emit_runtime_event(event: &str, payload: &[u8]) -> anyhow::Result<()> { + // Hold the system-wide lock across both the log append and register + // extension so separate processes cannot make their ordering diverge. + let _lock = runtime_event_lock()?; + let version = runtime_event_version()?; + let event = RuntimeEvent::new(event.to_string(), payload.to_vec(), version); + let mode = detect_tee_variant()?; if mode.has_tdx() { let digest = event.sha384_digest(); @@ -87,6 +174,11 @@ pub fn emit_runtime_event(event: &str, payload: &[u8]) -> anyhow::Result<()> { bank => anyhow::bail!("unsupported TPM PCR bank: {bank}"), } } + + // Commit the userspace log only after the platform register accepted the + // measurement. A device failure must never leave an unmeasured event in + // the trusted replay log. + event.emit().context("Failed to emit runtime event")?; Ok(()) } @@ -98,8 +190,8 @@ pub fn emit_runtime_event(event: &str, payload: &[u8]) -> anyhow::Result<()> { /// of producing quotes that can never verify. Does **not** append to the /// dstack event log / PCR14 lane — config is a separate register. pub fn measure_aws_config_pcr(config_id: &[u8; 48]) -> anyhow::Result<()> { - let mode = AttestationMode::detect()?; - if mode != AttestationMode::DstackAwsNitroTpm { + let mode = detect_tee_variant()?; + if mode != TeeVariant::DstackAwsNitroTpm { return Ok(()); } let config_pcr = u32::from(crate::attestation::AWS_NITRO_TPM_CONFIG_PCR); @@ -132,3 +224,36 @@ fn expected_aws_config_pcr(config_id: &[u8; 48]) -> [u8; 48] { material[48..].copy_from_slice(config_id); Sha384::digest(material).into() } + +#[cfg(test)] +mod runtime_event_version_tests { + use super::*; + + fn temp_path(name: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!("dstack-{name}-{}", std::process::id())) + } + + #[test] + fn rejects_conflicting_runtime_event_version() { + let path = temp_path("event-version-conflict"); + let _ = fs_err::remove_file(&path); + set_runtime_event_version_file(&path, EventLogVersion::V1).unwrap(); + set_runtime_event_version_file(&path, EventLogVersion::V1).unwrap(); + let err = set_runtime_event_version_file(&path, EventLogVersion::V2).unwrap_err(); + let message = err.to_string(); + assert!(message.contains("already set to 1"), "{message}"); + assert!( + message.contains("restart the CVM"), + "the conflict error must tell the operator how to apply a new version: {message}" + ); + let _ = fs_err::remove_file(path); + } + + #[test] + fn reports_unconfigured_runtime_event_version() { + let path = temp_path("event-version-missing"); + let _ = fs_err::remove_file(&path); + let err = runtime_event_version_file(path).unwrap_err(); + assert!(err.to_string().contains("complete dstack system setup")); + } +} diff --git a/dstack/dstack-attest/src/trust_anchors.rs b/dstack/dstack-attest/src/trust_anchors.rs new file mode 100644 index 000000000..fd2eb9b68 --- /dev/null +++ b/dstack/dstack-attest/src/trust_anchors.rs @@ -0,0 +1,207 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Trust anchors published inside a guest, and the checks that make them +//! trustworthy to read. +//! +//! A CVM must never let its host pick the trust anchor that verifies remote +//! attestation. The host sits outside the trust boundary, so a host-supplied +//! root would let it stand up a fake key provider and hand the guest keys it +//! never earned. +//! +//! An image that must verify non-production evidence still needs external +//! roots, so that handoff runs entirely inside the guest: `dstack-tee-simulator` +//! derives them from its seed and writes them to [`ANCHOR_DIR`], a tmpfs +//! directory the host cannot reach. Only the development image ships the +//! simulator, and image contents are measured, so on a production image nothing +//! ever creates that directory and the vendor production roots are the only +//! reachable outcome. +//! +//! [`crate::default_verifier`] is the only thing that should act on what +//! [`load_anchors`] returns. + +use std::{ + os::unix::fs::MetadataExt as _, + path::{Path, PathBuf}, +}; + +use anyhow::{bail, Context, Result}; + +use crate::attestation::RootCaPaths; + +/// Guest tmpfs directory carrying locally published trust anchors. +pub const ANCHOR_DIR: &str = "/run/dstack/attestation"; + +const ROOTS_FILE: &str = "roots.json"; + +/// Path of the published [`RootCaPaths`] within a trust anchor directory. +/// +/// The publisher writes it; [`load_anchors`] is the only reader. +pub fn roots_path(dir: &Path) -> PathBuf { + dir.join(ROOTS_FILE) +} + +/// Load trust anchors published inside this guest, if any. +/// +/// Returns `Ok(None)` when nothing published anchors, which is the only outcome +/// on a production image. +pub fn load_anchors(dir: &Path) -> Result<Option<RootCaPaths>> { + let path = roots_path(dir); + if !path.exists() { + return Ok(None); + } + ensure_owned_and_unwritable(dir, "trust anchor directory")?; + let meta = ensure_owned_and_unwritable(&path, "published roots")?; + if !meta.is_file() { + bail!("published roots is not a regular file"); + } + let root_ca: RootCaPaths = + serde_json::from_slice(&fs_err::read(&path).context("failed to read published roots")?) + .context("failed to parse published roots")?; + for root in [ + &root_ca.tdx, + &root_ca.gcp_tpm, + &root_ca.aws_nitro_enclave, + &root_ca.aws_nitro_tpm, + &root_ca.sev_snp_milan, + &root_ca.sev_snp_genoa, + &root_ca.sev_snp_turin, + ] + .into_iter() + .flatten() + { + // Confining every root to the published directory keeps a stale or + // tampered file from redirecting the verifier at a host-shared root. + if root.parent() != Some(dir) || root.file_name().is_none() { + bail!( + "trust anchor {} is outside {}", + root.display(), + dir.display() + ); + } + ensure_owned_and_unwritable(root, "trust anchor")?; + } + Ok(Some(root_ca)) +} + +/// Reject anything this process does not own or that others could rewrite. +/// +/// Symlink metadata, not the followed target: a symlink planted by another user +/// would otherwise pass the check while resolving somewhere unowned. +fn ensure_owned_and_unwritable(path: &Path, what: &str) -> Result<std::fs::Metadata> { + let meta = fs_err::symlink_metadata(path) + .with_context(|| format!("failed to stat {what} {}", path.display()))?; + let euid = rustix::process::geteuid().as_raw(); + if meta.uid() != euid { + bail!( + "{what} {} is owned by uid {} instead of {euid}", + path.display(), + meta.uid() + ); + } + if meta.mode() & 0o022 != 0 { + bail!( + "{what} {} is writable by group or others (mode {:o})", + path.display(), + meta.mode() & 0o7777 + ); + } + Ok(meta) +} + +#[cfg(test)] +mod tests { + use super::*; + // Mirrors what `crate::default_verifier` does with a loaded set, so the + // published roots are proven usable by the real verifier. + use crate::attestation::{AttestationVerifier, AttestationVerifierConfig}; + use std::os::unix::fs::PermissionsExt as _; + + fn sample_root() -> String { + let key = rcgen::KeyPair::generate_for(&rcgen::PKCS_ECDSA_P256_SHA256).unwrap(); + let mut params = rcgen::CertificateParams::new(vec![]).unwrap(); + params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + params.self_signed(&key).unwrap().pem() + } + + /// Stand in for the publisher, which lives in `dstack-tee-simulator`. + fn publish(dir: &Path, tdx_root: &str) -> RootCaPaths { + fs_err::create_dir_all(dir).unwrap(); + fs_err::set_permissions(dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + let root = dir.join("tdx-root-ca.pem"); + safe_write::safe_write_with_mode(&root, tdx_root.as_bytes(), 0o600).unwrap(); + let root_ca = RootCaPaths { + tdx: Some(root), + ..Default::default() + }; + write_roots(dir, &root_ca); + root_ca + } + + fn write_roots(dir: &Path, root_ca: &RootCaPaths) { + safe_write::safe_write_with_mode( + roots_path(dir), + serde_json::to_vec(root_ca).unwrap(), + 0o600, + ) + .unwrap(); + } + + fn verifier_for(root_ca: RootCaPaths) -> Result<AttestationVerifier> { + AttestationVerifier::load(&AttestationVerifierConfig { + insecure_allow_external_trust_anchors: true, + urls: Default::default(), + root_ca, + }) + } + + #[test] + fn absent_directory_selects_production_roots() { + let dir = tempfile::tempdir().unwrap(); + assert!(load_anchors(&dir.path().join("missing")).unwrap().is_none()); + } + + #[test] + fn published_roots_round_trip() { + let dir = tempfile::tempdir().unwrap(); + let dir = dir.path().join("attestation"); + let published = publish(&dir, &sample_root()); + + let root_ca = load_anchors(&dir).unwrap().expect("anchors should load"); + assert_eq!(root_ca.tdx, published.tdx); + assert_eq!(root_ca.gcp_tpm, None); + // What was published must be loadable by the real verifier. + verifier_for(root_ca).unwrap(); + } + + #[test] + fn a_malformed_root_fails_verifier_construction() { + let dir = tempfile::tempdir().unwrap(); + let dir = dir.path().join("attestation"); + publish(&dir, "not a certificate"); + let root_ca = load_anchors(&dir).unwrap().unwrap(); + assert!(verifier_for(root_ca).is_err()); + } + + #[test] + fn world_writable_roots_are_rejected() { + let dir = tempfile::tempdir().unwrap(); + let dir = dir.path().join("attestation"); + publish(&dir, &sample_root()); + fs_err::set_permissions(roots_path(&dir), std::fs::Permissions::from_mode(0o666)).unwrap(); + let error = load_anchors(&dir).unwrap_err().to_string(); + assert!(error.contains("writable by group or others"), "{error}"); + } + + #[test] + fn trust_anchor_outside_the_directory_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let anchors = dir.path().join("attestation"); + let mut root_ca = publish(&anchors, &sample_root()); + root_ca.tdx = Some(dir.path().join("host-shared-root.pem")); + write_roots(&anchors, &root_ca); + let error = load_anchors(&anchors).unwrap_err().to_string(); + assert!(error.contains("is outside"), "{error}"); + } +} diff --git a/dstack/dstack-attest/src/v1.rs b/dstack/dstack-attest/src/v1.rs index 7df502218..f895a94cf 100644 --- a/dstack/dstack-attest/src/v1.rs +++ b/dstack/dstack-attest/src/v1.rs @@ -92,7 +92,7 @@ impl PlatformEvidence { pub fn tdx_quote(&self) -> Option<&[u8]> { match self { Self::Tdx { quote, .. } | Self::GcpTdx { quote, .. } => Some(quote.as_slice()), - _ => None, + Self::NitroEnclave { .. } | Self::AwsNitroTpm { .. } | Self::SevSnp { .. } => None, } } @@ -101,42 +101,57 @@ impl PlatformEvidence { Self::Tdx { event_log, .. } | Self::GcpTdx { event_log, .. } => { Some(event_log.as_slice()) } - _ => None, + Self::NitroEnclave { .. } | Self::AwsNitroTpm { .. } | Self::SevSnp { .. } => None, } } pub fn tpm_quote(&self) -> Option<&TpmQuote> { match self { Self::GcpTdx { tpm_quote, .. } => Some(tpm_quote), - _ => None, + Self::Tdx { .. } + | Self::NitroEnclave { .. } + | Self::AwsNitroTpm { .. } + | Self::SevSnp { .. } => None, } } pub fn nsm_quote(&self) -> Option<&[u8]> { match self { Self::NitroEnclave { nsm_quote } => Some(nsm_quote.as_slice()), - _ => None, + Self::Tdx { .. } + | Self::GcpTdx { .. } + | Self::AwsNitroTpm { .. } + | Self::SevSnp { .. } => None, } } pub fn sev_snp_report(&self) -> Option<&[u8]> { match self { Self::SevSnp { report, .. } => Some(report.as_slice()), - _ => None, + Self::Tdx { .. } + | Self::GcpTdx { .. } + | Self::NitroEnclave { .. } + | Self::AwsNitroTpm { .. } => None, } } pub fn sev_snp_cert_chain(&self) -> Option<&[Vec<u8>]> { match self { Self::SevSnp { cert_chain, .. } => Some(cert_chain.as_slice()), - _ => None, + Self::Tdx { .. } + | Self::GcpTdx { .. } + | Self::NitroEnclave { .. } + | Self::AwsNitroTpm { .. } => None, } } pub fn sev_snp_mr_config_document(&self) -> Option<&str> { match self { Self::SevSnp { mr_config, .. } => Some(mr_config.as_str()), - _ => None, + Self::Tdx { .. } + | Self::GcpTdx { .. } + | Self::NitroEnclave { .. } + | Self::AwsNitroTpm { .. } => None, } } @@ -145,6 +160,13 @@ impl PlatformEvidence { .and_then(|document| MrConfigV3::from_document(document).ok()) } + pub fn tdx_event_log_mut(&mut self) -> Option<&mut Vec<TdxEvent>> { + match self { + Self::Tdx { event_log, .. } | Self::GcpTdx { event_log, .. } => Some(event_log), + Self::NitroEnclave { .. } | Self::AwsNitroTpm { .. } | Self::SevSnp { .. } => None, + } + } + pub fn into_stripped(self) -> Self { self.into_stripped_for_config("") } @@ -164,7 +186,11 @@ impl PlatformEvidence { event_log: strip_tdx_runtime_event_log(event_log), tpm_quote, }, - other => other, + // No TDX event log to strip. Listed rather than caught by a + // wildcard so that a new platform has to state its answer here. + evidence @ (Self::NitroEnclave { .. } + | Self::AwsNitroTpm { .. } + | Self::SevSnp { .. }) => evidence, } } } @@ -271,8 +297,17 @@ impl Attestation { } pub fn from_msgpack(bytes: &[u8]) -> Result<Self> { - let value: Self = - rmp_serde::from_slice(bytes).context("failed to decode attestation from msgpack")?; + let mut cursor = std::io::Cursor::new(bytes); + let mut decoder = rmp_serde::Deserializer::new(&mut cursor); + let value = + Self::deserialize(&mut decoder).context("failed to decode attestation from msgpack")?; + drop(decoder); + if cursor.position() != bytes.len() as u64 { + bail!( + "trailing bytes after attestation msgpack: {}", + bytes.len() as u64 - cursor.position() + ); + } if value.version != ATTESTATION_VERSION { bail!( "unsupported attestation version: expected {}, got {}", @@ -307,8 +342,23 @@ impl Attestation { stack: self.stack.into_dstack_pod(report_data_payload), } } +} - /// Return a new attestation with the report_data patched in both platform quote and stack. +/// Helpers for assembling synthetic attestations from a captured fixture. +/// +/// Simulator only. Patching the report data into a quote leaves the quote's +/// signature covering the bytes that were there before, so what comes out +/// cannot pass verification -- it is evidence-shaped, not evidence. That is +/// fine for a simulator whose callers skip verification, and wrong everywhere +/// else, so these are behind a feature that only the simulator turns on: a +/// per-package build of kms, gateway or verifier cannot reach them at all. +/// +/// (`cargo build --workspace` unifies features, so a workspace-wide build does +/// compile them into every crate. The gate is a statement of intent and a +/// backstop for the documented per-package release builds, not a hard wall.) +#[cfg(any(test, feature = "simulator"))] +impl Attestation { + /// Patch `report_data` into both the platform quote and the stack evidence. pub fn with_report_data(self, report_data: [u8; 64]) -> Self { use crate::attestation::{SNP_REPORT_DATA_RANGE, TDX_QUOTE_REPORT_DATA_RANGE}; @@ -322,6 +372,26 @@ impl Attestation { } PlatformEvidence::Tdx { quote, event_log } } + // Same TDX quote layout, so the same patch applies. The vTPM quote + // beside it cannot follow: its `qualified_data` is + // `sha256(tdx_quote)`, and re-deriving that would need the AK to + // re-sign. So this severs the TPM binding on top of the quote + // signature the patch already invalidates -- acceptable only + // because nothing verifies a simulator attestation. + PlatformEvidence::GcpTdx { + mut quote, + event_log, + tpm_quote, + } => { + if quote.len() >= TDX_QUOTE_REPORT_DATA_RANGE.end { + quote[TDX_QUOTE_REPORT_DATA_RANGE].copy_from_slice(&report_data); + } + PlatformEvidence::GcpTdx { + quote, + event_log, + tpm_quote, + } + } PlatformEvidence::SevSnp { mut report, cert_chain, @@ -336,8 +406,28 @@ impl Attestation { mr_config, } } - other => other, + // Nothing to patch: both carry their report data inside a signed + // document, and rewriting it would only invalidate the signature. + // Listed rather than caught by a wildcard so that a new platform + // has to state its answer here instead of silently getting this + // one -- GcpTdx was skipped for exactly that reason. + evidence @ (PlatformEvidence::NitroEnclave { .. } + | PlatformEvidence::AwsNitroTpm { .. }) => evidence, }; + Self { + version: self.version, + platform, + stack: self.stack, + } + .with_stack_report_data(report_data) + } + + /// Patch `report_data` into the stack evidence only, leaving the platform + /// quote untouched. + /// + /// For callers that go on to generate a real quote over the same report + /// data: patching the fixture's quote first would only be overwritten. + pub fn with_stack_report_data(self, report_data: [u8; 64]) -> Self { let stack = match self.stack { StackEvidence::Dstack { runtime_events, @@ -362,7 +452,7 @@ impl Attestation { }; Self { version: self.version, - platform, + platform: self.platform, stack, } } @@ -372,11 +462,14 @@ impl Attestation { mod tests { use super::*; use cc_eventlog::tdx::TDX_ACPI_DATA_EVENT_TYPE; + use dstack_types::mr_config::MrConfigV3; + use dstack_types::EventLogVersion; fn test_mr_config_document() -> String { MrConfigV3::new( vec![0x11; 20], vec![0x22; 32], + None, dstack_types::KeyProviderKind::None, Vec::new(), vec![0x33; 20], @@ -395,6 +488,8 @@ mod tests { digest: vec![0xaa, 0xbb, 0xcc], event: "pod".into(), event_payload: vec![0xde, 0xad, 0xbe, 0xef], + version: EventLogVersion::V1, + preimage: None, }], }, StackEvidence::DstackPod { @@ -402,6 +497,7 @@ mod tests { runtime_events: vec![RuntimeEvent { event: "pod".into(), payload: vec![0xca, 0xfe, 0xba, 0xbe], + version: EventLogVersion::V1, }], config: "{}".into(), report_data_payload: "{\"hello\":\"world\"}".into(), @@ -473,6 +569,8 @@ mod tests { digest: vec![idx as u8; 48], event: String::new(), event_payload: vec![0xff; idx + 1], + version: EventLogVersion::V1, + preimage: None, } } @@ -483,6 +581,8 @@ mod tests { digest: vec![idx as u8; 48], event: String::new(), event_payload: TDX_ACPI_DATA_EVENT_PAYLOAD.to_vec(), + version: EventLogVersion::V1, + preimage: None, } } @@ -490,6 +590,7 @@ mod tests { RuntimeEvent { event: "app-id".into(), payload: vec![0x42], + version: EventLogVersion::V1, } .into() } @@ -520,6 +621,41 @@ mod tests { assert_eq!(stripped[3].event_payload, vec![0x42]); } + #[test] + fn tdx_event_log_accessors_agree_on_gcp_tdx() { + // The mut accessor used to match bare TDX only, so `fill_v2_preimages` + // silently skipped GCP attestations and returned V2 events without the + // preimage the guest-agent proto promises clients can verify. Whatever + // the read accessor reaches, the mut one has to reach too. + let mut evidence = PlatformEvidence::GcpTdx { + quote: vec![0u8; 64], + event_log: vec![TdxEvent { + imr: 3, + event_type: cc_eventlog::DSTACK_RUNTIME_EVENT_TYPE, + digest: vec![0u8; 48], + event: "test-event".into(), + event_payload: b"payload".to_vec(), + version: EventLogVersion::V2, + preimage: None, + }], + tpm_quote: TpmQuote { + message: Vec::new(), + signature: Vec::new(), + pcr_values: Vec::new(), + ak_cert: Vec::new(), + platform: dstack_types::Platform::Gcp, + event_log: Vec::new(), + }, + }; + + assert!(evidence.tdx_event_log().is_some()); + let log = evidence + .tdx_event_log_mut() + .expect("mut accessor must reach GCP TDX too"); + cc_eventlog::tdx::fill_v2_preimages(log); + assert!(evidence.tdx_event_log().unwrap()[0].preimage.is_some()); + } + #[test] fn sev_snp_with_report_data_patches_report_and_stack() { let mut report = vec![0x11; 1184]; diff --git a/dstack/dstack-attest/tests/nitro_verify.rs b/dstack/dstack-attest/tests/nitro_verify.rs index 77be34938..5becbb36e 100644 --- a/dstack/dstack-attest/tests/nitro_verify.rs +++ b/dstack/dstack-attest/tests/nitro_verify.rs @@ -4,7 +4,9 @@ //! Integration test: verify Nitro Enclave attestation end-to-end -use dstack_attest::attestation::{AttestationQuote, DstackVerifiedReport, VersionedAttestation}; +use dstack_attest::attestation::{ + AttestationQuote, AttestationVerifier, DstackVerifiedReport, VersionedAttestation, +}; use nsm_qvl::{AttestationDocument, CoseSign1}; use std::time::{Duration, SystemTime}; @@ -40,8 +42,9 @@ async fn verify_nitro_attestation_bin() { } _ => panic!("unexpected quote type"), }; + let verifier = AttestationVerifier::new_prod(None).unwrap(); let verified = attestation - .verify_with_time(None, Some(fixed_now)) + .verify_with_time(&verifier, Some(fixed_now)) .await .unwrap(); let DstackVerifiedReport::DstackNitroEnclave(report) = verified.report else { diff --git a/dstack/dstack-attest/tests/sev_snp_verify.rs b/dstack/dstack-attest/tests/sev_snp_verify.rs index 8fa4c3317..38a19145c 100644 --- a/dstack/dstack-attest/tests/sev_snp_verify.rs +++ b/dstack/dstack-attest/tests/sev_snp_verify.rs @@ -102,7 +102,7 @@ fn verify_sev_snp_attestation_bin() { ); // The HOST_DATA-bound app identity is recovered from the mr_config document. assert_eq!( - hex::encode(&binding.mr_config.app_id), + hex::encode(binding.mr_config.app_id.as_deref().unwrap_or_default()), "86e59625be93207bc2351c4d1bba20037cec8e16", "mr_config app_id bound by HOST_DATA" ); @@ -271,6 +271,28 @@ fn forged_report_bytes_fail_signature_verification() { ); } +#[test] +fn tampered_real_amd_ask_fails_chain_verification() { + let report = fixture_report(); + let mut ask_der = pem::parse(SEV_ASK_PEM) + .expect("parse real AMD ASK") + .into_contents(); + let last = ask_der.last_mut().expect("ASK DER is non-empty"); + *last ^= 1; + let tampered_ask = pem::encode(&pem::Pem::new("CERTIFICATE", ask_der)); + + let error = verify_amd_snp_attestation(&AmdSnpAttestationInput { + report: &report, + ask_pem: tampered_ask.as_bytes(), + vcek_pem: SEV_VCEK_PEM, + }) + .expect_err("tampered AMD ASK must fail verification"); + assert!( + error.to_string().contains("cert chain verification"), + "unexpected error: {error:#}" + ); +} + #[test] fn wrong_collateral_is_rejected() { let report = fixture_report(); @@ -352,6 +374,7 @@ fn substituted_mr_config_breaks_host_data_binding() { let evil = MrConfigV3::new( vec![0xab; 20], vec![0xcd; 32], + None, KeyProviderKind::None, Vec::new(), vec![0xef; 20], diff --git a/dstack/dstack-attest/tests/snapshots/nitro_verify__app_info.snap b/dstack/dstack-attest/tests/snapshots/nitro_verify__app_info.snap index cc08badca..d62ea8cc0 100644 --- a/dstack/dstack-attest/tests/snapshots/nitro_verify__app_info.snap +++ b/dstack/dstack-attest/tests/snapshots/nitro_verify__app_info.snap @@ -1,6 +1,6 @@ --- source: dstack-attest/tests/nitro_verify.rs -assertion_line: 25 +assertion_line: 29 expression: app_info_str --- { @@ -11,5 +11,6 @@ expression: app_info_str "mr_system": "1894b0b29e94a9db16e88a2914f0923e52bb16c08bf3bb484df786a147e2eb79", "mr_aggregated": "1894b0b29e94a9db16e88a2914f0923e52bb16c08bf3bb484df786a147e2eb79", "os_image_hash": "1894b0b29e94a9db16e88a2914f0923e52bb16c08bf3bb484df786a147e2eb79", - "key_provider_info": "" + "key_provider_info": "", + "init_script_hashes": [] } diff --git a/dstack/dstack-mr/Cargo.toml b/dstack/dstack-mr/Cargo.toml index 125ba0c4e..7e3248486 100644 --- a/dstack/dstack-mr/Cargo.toml +++ b/dstack/dstack-mr/Cargo.toml @@ -35,6 +35,7 @@ bon.workspace = true log.workspace = true scale.workspace = true dstack-types.workspace = true +qemu-acpi.workspace = true [dev-dependencies] reqwest = { workspace = true, features = ["blocking"] } diff --git a/dstack/dstack-mr/cli/src/main.rs b/dstack/dstack-mr/cli/src/main.rs index 146112b3d..6f52081d3 100644 --- a/dstack/dstack-mr/cli/src/main.rs +++ b/dstack/dstack-mr/cli/src/main.rs @@ -2,10 +2,10 @@ // // SPDX-License-Identifier: Apache-2.0 -use anyhow::{Context, Result}; +use anyhow::{Context, Result, bail}; use clap::{Parser, Subcommand}; -use dstack_mr::{Machine, OvmfVariant, ovmf_variant_for_version}; -use dstack_types::ImageInfo; +use dstack_mr::{Machine, OvmfVariant}; +use dstack_types::{ImageInfo, VmConfig}; use fs_err as fs; use size_parser::parse_memory_size; use std::path::PathBuf; @@ -21,6 +21,9 @@ struct Cli { enum Commands { /// Measure a machine configuration Measure(MachineConfig), + /// Compute expected MRs from a `VmConfig` JSON and explain the RTMR0 event log entry by + /// entry. Optionally compare against actual MRTD/RTMR hex values from a quote. + Diagnose(DiagnoseConfig), } type Bool = bool; @@ -70,6 +73,14 @@ struct MachineConfig { #[arg(long, default_value = "1")] num_nics: u32, + /// Number of virtio-blk verity volumes + #[arg(long, default_value = "0")] + num_verity_volumes: u32, + + /// Attach a QEMU tpm-tis device backed by swtpm + #[arg(long, default_value = "false")] + swtpm: bool, + /// Disable hotplug #[arg(long, default_value = "false")] hotplug_off: Bool, @@ -82,11 +93,6 @@ struct MachineConfig { #[arg(long)] qemu_version: Option<String>, - /// dstack OS version (MAJOR.MINOR.PATCH), validated before using the supported OVMF - /// measurement layout. If omitted, falls back to `image_info.version`. - #[arg(long)] - dstack_os_version: Option<String>, - /// Output JSON #[arg(long)] json: bool, @@ -106,22 +112,11 @@ fn main() -> Result<()> { let firmware_path = parent_dir.join(&image_info.bios).display().to_string(); let kernel_path = parent_dir.join(&image_info.kernel).display().to_string(); let initrd_path = parent_dir.join(&image_info.initrd).display().to_string(); - let cmdline = image_info.cmdline + " initrd=initrd"; - - // CLI flag wins, then the explicit `ovmf_variant` in metadata.json, - // and finally the OS version field. Older metadata.json files may - // carry neither, in which case fall back to the default. - let ovmf_variant = if let Some(v) = config.dstack_os_version.as_deref() { - ovmf_variant_for_version(v) - .with_context(|| format!("invalid dstack OS version: {v}"))? - } else if let Some(variant) = image_info.ovmf_variant { - variant - } else if !image_info.version.is_empty() { - ovmf_variant_for_version(&image_info.version) - .with_context(|| format!("invalid dstack OS version: {}", image_info.version))? - } else { - OvmfVariant::default() - }; + let cmdline = dstack_mr::tdx::measured_kernel_cmdline(&image_info.cmdline); + + // The image declares its OVMF layout. Older metadata.json files + // predate the field, so fall back to the only layout that existed. + let ovmf_variant = image_info.ovmf_variant.unwrap_or_default(); let machine = Machine::builder() .cpu_count(config.cpu) @@ -131,6 +126,7 @@ fn main() -> Result<()> { .initrd(&initrd_path) .kernel_cmdline(&cmdline) .maybe_two_pass_add_pages(config.two_pass_add_pages) + .normalized_setup_header(image_info.kernel_header_normalized) .maybe_pic(config.pic) .smm(config.smm) .maybe_pci_hole64_size(config.pci_hole64_size) @@ -138,6 +134,8 @@ fn main() -> Result<()> { .num_gpus(config.num_gpus) .num_nvswitches(config.num_nvswitches) .num_nics(config.num_nics) + .num_verity_volumes(config.num_verity_volumes) + .swtpm(config.swtpm) .hotplug_off(config.hotplug_off) .root_verity(config.root_verity) .maybe_qemu_version(config.qemu_version.clone()) @@ -158,6 +156,292 @@ fn main() -> Result<()> { println!("RTMR2: {}", hex::encode(measurements.rtmr2)); } } + Commands::Diagnose(config) => run_diagnose(config)?, + } + + Ok(()) +} + +#[derive(Parser)] +struct DiagnoseConfig { + /// VmConfig JSON. Matches the schema VMM serializes into KMS metadata + /// (dstack_types::VmConfig). When KMS/verifier reports an MR mismatch, dump + /// the same VmConfig payload it used and pass it here. + #[arg(long)] + vm_config: PathBuf, + + /// Image directory containing ovmf.fd / bzImage / initramfs.cpio.gz / + /// metadata.json. If omitted, falls back to looking up `vm_config.image` + /// under `--image-base-dir`. + #[arg(long)] + image_dir: Option<PathBuf>, + + /// Base directory containing one subdir per image (e.g. + /// /opt/dstack/dstack-images). Only used when `--image-dir` is not given. + #[arg(long)] + image_base_dir: Option<PathBuf>, + + /// Optional actual measurements for comparison. Hex strings (no `0x` prefix). + #[arg(long)] + actual_mrtd: Option<String>, + #[arg(long)] + actual_rtmr0: Option<String>, + #[arg(long)] + actual_rtmr1: Option<String>, + #[arg(long)] + actual_rtmr2: Option<String>, + + /// Actual quote event log JSON used to identify the first divergent RTMR0 event. + #[arg(long)] + actual_event_log: Option<PathBuf>, + + /// Output JSON + #[arg(long)] + json: bool, +} + +/// Semantic label for each RTMR0 event log entry. Indices match +/// `tdvf::rtmr0_log` (see dstack-mr/src/tdvf.rs). +fn rtmr0_labels(_variant: OvmfVariant) -> &'static [(&'static str, &'static str)] { + &[ + ( + "td_hob", + "varies-with: memory_size, firmware section layout", + ), + ("cfv_image", "fixed: hardcoded constant"), + ("efi:SecureBoot", "fixed: TDX EFI variable"), + ("efi:PK", "fixed: TDX EFI variable"), + ("efi:KEK", "fixed: TDX EFI variable"), + ("efi:db", "fixed: TDX EFI variable"), + ("efi:dbx", "fixed: TDX EFI variable"), + ("separator", "fixed: sha384(0x00000000)"), + ( + "acpi_loader", + "varies-with: cpu_count, pic, smm, hpet, hotplug_off, pci_hole64, root_verity, host_share_mode, num_gpus, num_nvswitches, hugepages, qemu_version", + ), + ("acpi_rsdp", "same as acpi_loader"), + ("acpi_tables", "same as acpi_loader"), + ( + "boot_order", + "fixed: sha384(0x0000) — raw 2 bytes in legacy OVMF", + ), + ("Boot0000", "fixed: legacy OVMF UiApp constant"), + ] +} + +fn resolve_image_dir(config: &DiagnoseConfig, vm: &VmConfig) -> Result<PathBuf> { + if let Some(dir) = &config.image_dir { + return Ok(dir.clone()); + } + let base = config + .image_base_dir + .as_ref() + .context("either --image-dir or --image-base-dir must be set")?; + let image_name = vm + .image + .as_ref() + .context("vm_config.image is empty; pass --image-dir directly")?; + Ok(base.join(image_name)) +} + +fn check(label: &str, expected: &[u8], actual_hex: &Option<String>) -> Option<bool> { + actual_hex.as_ref().map(|hex_str| { + let trimmed = hex_str.trim().trim_start_matches("0x"); + match hex::decode(trimmed) { + Ok(actual) if actual == expected => { + println!(" {label}: MATCH"); + true + } + Ok(actual) => { + println!( + " {label}: MISMATCH\n expected: {}\n actual: {}", + hex::encode(expected), + hex::encode(&actual), + ); + false + } + Err(e) => { + eprintln!(" {label}: invalid actual hex ({e})"); + false + } + } + }) +} + +fn compare_rtmr0_event_log( + expected: &[Vec<u8>], + actual_path: &PathBuf, + labels: &[(&str, &str)], +) -> Result<bool> { + let raw = fs::read_to_string(actual_path).context("failed to read --actual-event-log")?; + let events: Vec<serde_json::Value> = + serde_json::from_str(&raw).context("failed to parse actual event log JSON")?; + let actual: Vec<Vec<u8>> = events + .iter() + .filter(|event| event.get("imr").and_then(serde_json::Value::as_u64) == Some(0)) + .map(|event| { + let digest = event + .get("digest") + .and_then(serde_json::Value::as_str) + .context("RTMR0 event has no digest")?; + hex::decode(digest).context("RTMR0 event digest is not hex") + }) + .collect::<Result<_>>()?; + let count = expected.len().max(actual.len()); + for index in 0..count { + if expected.get(index) == actual.get(index) { + continue; + } + let (label, _) = labels.get(index).copied().unwrap_or(("(unlabelled)", "")); + println!( + " FIRST DIVERGENT RTMR0 EVENT: index={index} label={label}\n expected: {}\n actual: {}", + expected + .get(index) + .map(hex::encode) + .unwrap_or_else(|| "<missing>".to_string()), + actual + .get(index) + .map(hex::encode) + .unwrap_or_else(|| "<missing>".to_string()), + ); + return Ok(false); + } + println!(" RTMR0 EVENT LOG: MATCH"); + Ok(true) +} + +fn run_diagnose(config: &DiagnoseConfig) -> Result<()> { + let raw = fs::read_to_string(&config.vm_config).context("failed to read --vm-config")?; + let vm: VmConfig = serde_json::from_str(&raw).context("failed to parse VmConfig JSON")?; + + let image_dir = resolve_image_dir(config, &vm)?; + let metadata_path = image_dir.join("metadata.json"); + let metadata = fs::read_to_string(&metadata_path) + .with_context(|| format!("failed to read {}", metadata_path.display()))?; + let image_info: ImageInfo = serde_json::from_str(&metadata)?; + + let firmware = image_dir.join(&image_info.bios).display().to_string(); + let kernel = image_dir.join(&image_info.kernel).display().to_string(); + let initrd = image_dir.join(&image_info.initrd).display().to_string(); + let cmdline = dstack_mr::tdx::measured_kernel_cmdline(&image_info.cmdline); + + // Same resolution order as the verifier (see verifier::compute_measurement_details): + // explicit vm_config.ovmf_variant > image_info.ovmf_variant > legacy default. + let ovmf_variant = vm + .ovmf_variant + .or(image_info.ovmf_variant) + .unwrap_or_default(); + + let details = Machine::builder() + .cpu_count(vm.cpu_count) + .memory_size(vm.memory_size) + .firmware(&firmware) + .kernel(&kernel) + .initrd(&initrd) + .kernel_cmdline(&cmdline) + .root_verity(true) + .hotplug_off(vm.hotplug_off) + .maybe_two_pass_add_pages(vm.qemu_single_pass_add_pages) + .normalized_setup_header(image_info.kernel_header_normalized) + .maybe_pic(vm.pic) + .maybe_qemu_version(vm.qemu_version.clone()) + .maybe_pci_hole64_size(if vm.pci_hole64_size > 0 { + Some(vm.pci_hole64_size) + } else { + None + }) + .hugepages(vm.hugepages) + .num_gpus(vm.num_gpus) + .num_nvswitches(vm.num_nvswitches) + .host_share_mode(vm.host_share_mode.clone()) + .ovmf_variant(ovmf_variant) + .build() + .measure_with_logs() + .context("failed to compute expected MRs")?; + + let labels = rtmr0_labels(ovmf_variant); + + if config.json { + let log: Vec<serde_json::Value> = details.rtmr_logs[0] + .iter() + .enumerate() + .map(|(i, h)| { + let (label, note) = labels.get(i).copied().unwrap_or(("(unlabelled)", "")); + serde_json::json!({ + "index": i, + "label": label, + "digest": hex::encode(h), + "note": note, + }) + }) + .collect(); + let out = serde_json::json!({ + "ovmf_variant": format!("{:?}", ovmf_variant), + "mrtd": hex::encode(&details.measurements.mrtd), + "rtmr0": hex::encode(&details.measurements.rtmr0), + "rtmr1": hex::encode(&details.measurements.rtmr1), + "rtmr2": hex::encode(&details.measurements.rtmr2), + "rtmr0_log": log, + }); + println!("{}", serde_json::to_string_pretty(&out)?); + return Ok(()); + } + + println!("=== inputs ==="); + println!( + " cpu={} mem={} qemu_version={:?} pic={:?} two_pass={:?}", + vm.cpu_count, vm.memory_size, vm.qemu_version, vm.pic, vm.qemu_single_pass_add_pages, + ); + println!( + " hugepages={} num_gpus={} num_nvswitches={} hotplug_off={} pci_hole64={}", + vm.hugepages, vm.num_gpus, vm.num_nvswitches, vm.hotplug_off, vm.pci_hole64_size, + ); + println!(" host_share_mode={:?}", vm.host_share_mode); + println!(" image_dir={}", image_dir.display()); + println!(" ovmf_variant={:?}", ovmf_variant); + + println!("\n=== expected measurements ==="); + println!(" MRTD: {}", hex::encode(&details.measurements.mrtd)); + println!(" RTMR0: {}", hex::encode(&details.measurements.rtmr0)); + println!(" RTMR1: {}", hex::encode(&details.measurements.rtmr1)); + println!(" RTMR2: {}", hex::encode(&details.measurements.rtmr2)); + + println!( + "\n=== RTMR0 event log ({} entries) ===", + details.rtmr_logs[0].len() + ); + for (i, hash) in details.rtmr_logs[0].iter().enumerate() { + let (label, note) = labels.get(i).copied().unwrap_or(("(unlabelled)", "")); + println!(" [{:>2}] {:<20} {}", i, label, hex::encode(hash)); + if !note.is_empty() { + println!(" {note}"); + } + } + + let want_compare = config.actual_mrtd.is_some() + || config.actual_rtmr0.is_some() + || config.actual_rtmr1.is_some() + || config.actual_rtmr2.is_some() + || config.actual_event_log.is_some(); + if want_compare { + println!("\n=== comparison ==="); + let mut all_ok = true; + if let Some(actual_event_log) = &config.actual_event_log { + all_ok &= compare_rtmr0_event_log(&details.rtmr_logs[0], actual_event_log, labels)?; + } + for (label, expected, actual) in [ + ("MRTD ", &details.measurements.mrtd, &config.actual_mrtd), + ("RTMR0", &details.measurements.rtmr0, &config.actual_rtmr0), + ("RTMR1", &details.measurements.rtmr1, &config.actual_rtmr1), + ("RTMR2", &details.measurements.rtmr2, &config.actual_rtmr2), + ] { + if let Some(ok) = check(label, expected, actual) { + all_ok &= ok; + } + } + if !all_ok { + bail!("one or more measurements mismatched"); + } } Ok(()) diff --git a/dstack/dstack-mr/src/acpi.rs b/dstack/dstack-mr/src/acpi.rs index db8f52845..00a544f13 100644 --- a/dstack/dstack-mr/src/acpi.rs +++ b/dstack/dstack-mr/src/acpi.rs @@ -2,18 +2,13 @@ // // SPDX-License-Identifier: Apache-2.0 -//! This module provides functionality to generate ACPI tables for QEMU, -//! translated from an original Go implementation. +//! QEMU-compatible ACPI table generation for TDX measurement. use anyhow::{bail, Context, Result}; -use log::debug; -use scale::Decode; +use qemu_acpi::{MachineConfig, QemuVersion}; use crate::Machine; -const LDR_LENGTH: usize = 4096; -const FIXED_STRING_LEN: usize = 56; - #[derive(Debug, Clone)] pub struct Tables { pub tables: Vec<u8>, @@ -22,443 +17,37 @@ pub struct Tables { } impl Machine<'_> { - fn create_tables(&self) -> Result<Vec<u8>> { - if self.cpu_count == 0 { - bail!("cpuCount must be greater than 0"); - } - let mem_size_mb = self.memory_size / (1024 * 1024); - - // Dummy disk and shared directory. Use as placeholders for the qemu arguments. - let dummy_disk = "/bin/sh"; - let shared_dir = "/bin"; - - // Prepare the command arguments - let mut cmd = std::process::Command::new("dstack-acpi-tables"); - cmd.args([ - "-cpu", - "qemu64", - "-smp", - &self.cpu_count.to_string(), - "-m", - &format!("{mem_size_mb}M"), - "-nographic", - "-nodefaults", - "-serial", - "stdio", - "-bios", - self.firmware, - "-kernel", - self.kernel, - "-initrd", - dummy_disk, - "-drive", - &format!("file={dummy_disk},if=none,id=hd1,format=raw,readonly=on"), - "-device", - "virtio-blk-pci,drive=hd1", - ]); - - // One virtio-net-pci per NIC. Emitted at the same position and, for the - // default single-NIC case, with the exact same args as the previous - // hardcoded layout so RTMR0 stays byte-for-byte unchanged. - for i in 0..self.num_nics { - cmd.arg("-netdev").arg(format!("user,id=net{i}")); - cmd.arg("-device") - .arg(format!("virtio-net-pci,netdev=net{i}")); - } - - cmd.args([ - "-object", - "tdx-guest,id=tdx", - "-device", - "vhost-vsock-pci,guest-cid=3", - ]); - - // Configure shared files delivery: either via disk or 9p - match self.host_share_mode.as_str() { - "" | "9p" => { - // Use 9p virtfs (default) - cmd.args([ - "-virtfs", - &format!( - "local,path={shared_dir},mount_tag=host-shared,readonly=on,security_model=none,id=virtfs0", - ), - ]); - } - "vvfat" | "vhd" => { - // Use a second virtual disk (hd2) to share files - cmd.args([ - "-drive", - &format!("file={dummy_disk},if=none,id=hd2,format=raw,readonly=on"), - "-device", - "virtio-blk-pci,drive=hd2", - ]); - } - _ => { - bail!("Invalid shared disk mode: {}", self.host_share_mode); - } - } - - if self.root_verity { - cmd.args([ - "-drive", - &format!("file={dummy_disk},if=none,id=hd0,format=raw,readonly=on"), - "-device", - "virtio-blk-pci,drive=hd0", - ]); - } else { - cmd.args(["-cdrom", dummy_disk]); - } - - let mut machine = - "q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off".to_string(); - if self.smm { - machine.push_str(",smm=on"); - } else { - machine.push_str(",smm=off"); - } - - let vopt = self - .versioned_options() - .context("Failed to get versioned options")?; - - if vopt.pic { - machine.push_str(",pic=on"); - } else { - machine.push_str(",pic=off"); - } - cmd.args(["-machine", &machine]); - if self.hugepages { - let cpu_end = self.cpu_count - 1; - cmd.args([ - "-numa", - &format!("node,nodeid=0,cpus=0-{cpu_end},memdev=mem0"), - "-object", - &format!("memory-backend-file,id=mem0,size={mem_size_mb}M,mem-path=/dev/hugepages,share=on,prealloc=no,host-nodes=0,policy=bind"), - ]); - } - let mut port_num = 0; - if self.num_gpus > 0 { - cmd.args(["-object", "iommufd,id=iommufd0"]); - let bus = if self.hugepages { - cmd.args([ - "-device", - "pxb-pcie,id=pcie.node0,bus=pcie.0,addr=10,numa_node=0,bus_nr=5", - ]); - "pcie.node0" - } else { - "pcie.0" - }; - for _ in 0..self.num_gpus { - cmd.args([ - "-device", - &format!("pcie-root-port,id=pci.{port_num},bus={bus},chassis={port_num}"), - "-device", - &format!("vfio-pci,host=00:00.0,bus=pci.{port_num},iommufd=iommufd0"), - ]); - port_num += 1; - } - } - - for _ in 0..self.num_nvswitches { - cmd.args([ - "-device", - &format!("pcie-root-port,id=pci.{port_num},bus=pcie.0,chassis={port_num}"), - "-device", - &format!("vfio-pci,host=00:00.0,bus=pci.{port_num},iommufd=iommufd0"), - ]); - port_num += 1; - } - - if self.hotplug_off { - cmd.args([ - "-global", - "ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off", - ]); - } - if let Some(pci_hole64_size) = self.pci_hole64_size { - cmd.args([ - "-global", - &format!("q35-pcihost.pci-hole64-size=0x{:x}", pci_hole64_size), - ]); - } - - debug!("qemu command: {cmd:?}"); - - let ver = vopt.version; - // Execute the command and capture output - let output = cmd - .env( - "QEMU_ACPI_COMPAT_VER", - format!("{}.{}.{}", ver.0, ver.1, ver.2), - ) - .output() - .context("failed to execute dstack-acpi-tables")?; - - // Check if the command was successful - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("dstack-acpi-tables failed: {stderr}"); - } - Ok(output.stdout) - } - pub fn build_tables(&self) -> Result<Tables> { - let tpl = self.create_tables()?; - // Find all required ACPI tables - let (dsdt_offset, dsdt_csum, dsdt_len) = find_acpi_table(&tpl, "DSDT")?; - let (facp_offset, facp_csum, facp_len) = find_acpi_table(&tpl, "FACP")?; - let (apic_offset, apic_csum, apic_len) = find_acpi_table(&tpl, "APIC")?; - let (mcfg_offset, mcfg_csum, mcfg_len) = find_acpi_table(&tpl, "MCFG")?; - let (waet_offset, waet_csum, waet_len) = find_acpi_table(&tpl, "WAET")?; - let (rsdt_offset, rsdt_csum, rsdt_len) = find_acpi_table(&tpl, "RSDT")?; - - // Generate RSDP - let mut rsdp = Vec::with_capacity(20); - rsdp.extend_from_slice(b"RSD PTR "); // Signature - rsdp.push(0x00); // Checksum placeholder - rsdp.extend_from_slice(b"BOCHS "); // OEM ID - rsdp.push(0x00); // Revision - rsdp.extend_from_slice(&rsdt_offset.to_le_bytes()); // RSDT Address - - // Generate table loader commands - let mut ldr = TableLoader::new(); - ldr.append(LoaderCmd::Allocate { - file: "etc/acpi/rsdp", - alignment: 16, - zone: 2, - }); - ldr.append(LoaderCmd::Allocate { - file: "etc/acpi/tables", - alignment: 64, - zone: 1, - }); - ldr.append(LoaderCmd::AddChecksum { - file: "etc/acpi/tables", - result_offset: dsdt_csum, - start: dsdt_offset, - length: dsdt_len, - }); - ldr.append(LoaderCmd::AddPtr { - pointer_file: "etc/acpi/tables", - pointee_file: "etc/acpi/tables", - pointer_offset: facp_offset + 36, - pointer_size: 4, - }); - ldr.append(LoaderCmd::AddPtr { - pointer_file: "etc/acpi/tables", - pointee_file: "etc/acpi/tables", - pointer_offset: facp_offset + 40, - pointer_size: 4, - }); - ldr.append(LoaderCmd::AddPtr { - pointer_file: "etc/acpi/tables", - pointee_file: "etc/acpi/tables", - pointer_offset: facp_offset + 140, - pointer_size: 8, - }); - ldr.append(LoaderCmd::AddChecksum { - file: "etc/acpi/tables", - result_offset: facp_csum, - start: facp_offset, - length: facp_len, - }); - ldr.append(LoaderCmd::AddChecksum { - file: "etc/acpi/tables", - result_offset: apic_csum, - start: apic_offset, - length: apic_len, - }); - ldr.append(LoaderCmd::AddChecksum { - file: "etc/acpi/tables", - result_offset: mcfg_csum, - start: mcfg_offset, - length: mcfg_len, - }); - ldr.append(LoaderCmd::AddChecksum { - file: "etc/acpi/tables", - result_offset: waet_csum, - start: waet_offset, - length: waet_len, - }); - ldr.append(LoaderCmd::AddPtr { - pointer_file: "etc/acpi/tables", - pointee_file: "etc/acpi/tables", - pointer_offset: rsdt_offset + 36, - pointer_size: 4, - }); - ldr.append(LoaderCmd::AddPtr { - pointer_file: "etc/acpi/tables", - pointee_file: "etc/acpi/tables", - pointer_offset: rsdt_offset + 40, - pointer_size: 4, - }); - ldr.append(LoaderCmd::AddPtr { - pointer_file: "etc/acpi/tables", - pointee_file: "etc/acpi/tables", - pointer_offset: rsdt_offset + 44, - pointer_size: 4, - }); - ldr.append(LoaderCmd::AddPtr { - pointer_file: "etc/acpi/tables", - pointee_file: "etc/acpi/tables", - pointer_offset: rsdt_offset + 48, - pointer_size: 4, - }); - ldr.append(LoaderCmd::AddChecksum { - file: "etc/acpi/tables", - result_offset: rsdt_csum, - start: rsdt_offset, - length: rsdt_len, - }); - ldr.append(LoaderCmd::AddPtr { - pointer_file: "etc/acpi/rsdp", - pointee_file: "etc/acpi/tables", - pointer_offset: 16, - pointer_size: 4, - }); - ldr.append(LoaderCmd::AddChecksum { - file: "etc/acpi/rsdp", - result_offset: 8, - start: 0, - length: 20, - }); - // 8. Pad the loader command blob to the required length - if ldr.buffer.len() < LDR_LENGTH { - ldr.buffer.resize(LDR_LENGTH, 0); + if self.swtpm { + bail!("swtpm measurement is not supported"); } - + let options = self + .versioned_options() + .context("failed to get QEMU-versioned options")?; + match self.host_share_mode.as_str() { + "" | "9p" | "vvfat" | "vhd" => {} + value => bail!("invalid shared disk mode: {value}"), + } + let config = MachineConfig { + qemu_version: QemuVersion::new(options.version.0, options.version.1, options.version.2), + cpu_count: self.cpu_count, + memory_size: self.memory_size, + pic: options.pic, + smm: self.smm, + hugepages: self.hugepages, + num_gpus: self.num_gpus, + num_nvswitches: self.num_nvswitches, + num_nics: self.num_nics, + num_verity_volumes: self.num_verity_volumes, + hotplug_off: self.hotplug_off, + root_verity: self.root_verity, + pci_hole64_size: self.pci_hole64_size, + }; + let blobs = qemu_acpi::build(&config).context("failed to generate QEMU ACPI tables")?; Ok(Tables { - tables: tpl, - rsdp, - loader: ldr.buffer, + tables: blobs.tables, + rsdp: blobs.rsdp, + loader: blobs.loader, }) } } - -/// An enum to represent the different QEMU loader commands in a type-safe way. -#[derive(Debug)] -enum LoaderCmd<'a> { - Allocate { - file: &'a str, - alignment: u32, - zone: u8, - }, - AddPtr { - pointer_file: &'a str, - pointee_file: &'a str, - pointer_offset: u32, - pointer_size: u8, - }, - AddChecksum { - file: &'a str, - result_offset: u32, - start: u32, - length: u32, - }, -} - -struct TableLoader { - buffer: Vec<u8>, -} - -impl TableLoader { - fn new() -> Self { - Self { - buffer: Vec::with_capacity(LDR_LENGTH), - } - } - fn append(&mut self, cmd: LoaderCmd) { - qemu_loader_append(&mut self.buffer, cmd); - } -} - -/// Appends a fixed-length, null-padded string to the data buffer. -fn append_fixed_string(data: &mut Vec<u8>, s: &str) { - let mut s_bytes = s.as_bytes().to_vec(); - s_bytes.resize(FIXED_STRING_LEN, 0); - data.extend_from_slice(&s_bytes); -} - -/// Appends a serialized QEMU loader command to the data buffer. -fn qemu_loader_append(data: &mut Vec<u8>, cmd: LoaderCmd) { - match cmd { - LoaderCmd::Allocate { - file, - alignment, - zone, - } => { - data.extend_from_slice(&[0x01, 0x00, 0x00, 0x00]); - append_fixed_string(data, file); - data.extend_from_slice(&alignment.to_le_bytes()); - data.push(zone); - data.resize(data.len() + 63, 0); // Padding - } - LoaderCmd::AddPtr { - pointer_file, - pointee_file, - pointer_offset, - pointer_size, - } => { - data.extend_from_slice(&[0x02, 0x00, 0x00, 0x00]); - append_fixed_string(data, pointer_file); - append_fixed_string(data, pointee_file); - data.extend_from_slice(&pointer_offset.to_le_bytes()); - data.push(pointer_size); - data.resize(data.len() + 7, 0); // Padding - } - LoaderCmd::AddChecksum { - file, - result_offset, - start, - length, - } => { - data.extend_from_slice(&[0x03, 0x00, 0x00, 0x00]); - append_fixed_string(data, file); - data.extend_from_slice(&result_offset.to_le_bytes()); - data.extend_from_slice(&start.to_le_bytes()); - data.extend_from_slice(&length.to_le_bytes()); - data.resize(data.len() + 56, 0); // Padding - } - } -} - -/// ACPI table header (first 8 bytes of every ACPI table) -#[derive(Debug, Decode)] -struct AcpiTableHeader { - signature: [u8; 4], - length: u32, -} - -/// Searches for an ACPI table with the given signature and returns its offset, -/// checksum offset, and length. -fn find_acpi_table(tables: &[u8], signature: &str) -> Result<(u32, u32, u32)> { - let sig_bytes = signature.as_bytes(); - if sig_bytes.len() != 4 { - bail!("Signature must be 4 bytes long, but got '{signature}'"); - } - - let mut offset = 0; - while offset < tables.len() { - // Ensure there's enough space for a table header - if offset + 8 > tables.len() { - bail!("Table not found: {signature}"); - } - - let header = AcpiTableHeader::decode(&mut &tables[offset..]) - .context("failed to decode ACPI table header")?; - - if header.signature == sig_bytes { - // Found the table - return Ok((offset as u32, (offset + 9) as u32, header.length)); - } - - if header.length == 0 { - // Invalid table length, stop searching - bail!("found table with zero length at offset {offset}"); - } - // Move to the next table - offset += header.length as usize; - } - - bail!("table not found: {signature}"); -} diff --git a/dstack/dstack-mr/src/kernel.rs b/dstack/dstack-mr/src/kernel.rs index a4e969563..7a130321d 100644 --- a/dstack/dstack-mr/src/kernel.rs +++ b/dstack/dstack-mr/src/kernel.rs @@ -237,15 +237,35 @@ pub(crate) fn patched_kernel_authenticode_sha384( authenticode_sha384_hash(&kd).context("Failed to compute kernel hash") } -/// Measures a QEMU-patched TDX kernel image. +/// Compute the first RTMR[1] event digest for an image whose OVMF normalizes +/// the Linux setup header: the Authenticode SHA-384 of the kernel file itself. +/// +/// Both sides zero the boot-loader-written fields -- +/// `os/image/normalize-kernel-header.py` in the image build and +/// `0007-OvmfPkg-QemuKernelLoaderFsDxe-normalize-setup-header.patch` in the +/// firmware -- so what OVMF measures is the file, on every QEMU version and at +/// every guest memory size. +pub(crate) fn kernel_authenticode_sha384(kernel_data: &[u8]) -> Result<Vec<u8>> { + authenticode_sha384_hash(kernel_data).context("failed to compute kernel hash") +} + +/// Measures the TDX kernel image OVMF loads. +/// +/// `normalized_setup_header` says which kernel bytes OVMF ends up measuring: +/// the kernel file as shipped when the image's firmware normalizes the setup +/// header, otherwise QEMU's rewritten copy. pub(crate) fn rtmr1_log( kernel_data: &[u8], initrd_size: u32, mem_size: u64, acpi_data_size: u32, + normalized_setup_header: bool, ) -> Result<Vec<Vec<u8>>> { - let kernel_hash = - patched_kernel_authenticode_sha384(kernel_data, initrd_size, mem_size, acpi_data_size)?; + let kernel_hash = if normalized_setup_header { + kernel_authenticode_sha384(kernel_data)? + } else { + patched_kernel_authenticode_sha384(kernel_data, initrd_size, mem_size, acpi_data_size)? + }; Ok(vec![ kernel_hash, measure_sha384(b"Calling EFI Application from Boot Option"), @@ -270,6 +290,56 @@ mod tests { u32::from_le_bytes(kernel[0x218..0x21c].try_into().unwrap()) } + /// A minimal PE/COFF so `authenticode_sha384_hash` has something to walk. + fn pe_kernel() -> Vec<u8> { + let mut kernel = vec![0u8; 0x2000]; + let lfanew = 0x40usize; + kernel[0x3c..0x40].copy_from_slice(&(lfanew as u32).to_le_bytes()); + kernel[lfanew..lfanew + 4].copy_from_slice(&object::pe::IMAGE_NT_SIGNATURE.to_le_bytes()); + let coff = lfanew + 4; + // SizeOfOptionalHeader, then PE32+ magic and SizeOfHeaders. + kernel[coff + 16..coff + 18].copy_from_slice(&0xf0u16.to_le_bytes()); + let opt = coff + 20; + kernel[opt..opt + 2].copy_from_slice(&0x020bu16.to_le_bytes()); + kernel[opt + 60..opt + 64].copy_from_slice(&0x400u32.to_le_bytes()); + // Setup header: protocol 2.12, and a non-zero heap_end_ptr like a real + // build has, so the two branches cannot coincide by accident. + kernel[0x202..0x206].copy_from_slice(b"HdrS"); + kernel[0x206..0x208].copy_from_slice(&0x020cu16.to_le_bytes()); + // XLF_CAN_BE_LOADED_ABOVE_4G, so QEMU derives the initrd address from + // available low memory and the patched digest moves with guest RAM. + kernel[0x236..0x238].copy_from_slice(&0x0040u16.to_le_bytes()); + kernel[0x224..0x226].copy_from_slice(&0x50a0u16.to_le_bytes()); + kernel + } + + /// The flag has to reach the digest, not just the struct: an image whose + /// firmware normalizes must measure the file, and one whose firmware does + /// not must measure QEMU's rewritten copy. + #[test] + fn the_normalized_flag_selects_which_kernel_bytes_are_measured() { + let kernel = pe_kernel(); + let normalized = rtmr1_log(&kernel, 0x1000, 0x8000_0000, 0x28000, true).unwrap(); + let patched = rtmr1_log(&kernel, 0x1000, 0x8000_0000, 0x28000, false).unwrap(); + assert_ne!(normalized[0], patched[0]); + assert_eq!(normalized[0], kernel_authenticode_sha384(&kernel).unwrap()); + assert_eq!( + patched[0], + patched_kernel_authenticode_sha384(&kernel, 0x1000, 0x8000_0000, 0x28000).unwrap() + ); + } + + /// Normalizing is what makes the digest independent of guest RAM, so the + /// two branches must disagree about that too. + #[test] + fn only_the_patched_digest_moves_with_guest_memory() { + let kernel = pe_kernel(); + let at = |mem| rtmr1_log(&kernel, 0x1000, mem, 0x28000, true).unwrap()[0].clone(); + assert_eq!(at(0x8000_0000), at(0xA000_0000)); + let patched_at = |mem| rtmr1_log(&kernel, 0x1000, mem, 0x28000, false).unwrap()[0].clone(); + assert_ne!(patched_at(0x8000_0000), patched_at(0xA000_0000)); + } + #[test] fn tdx_kernel_patch_uses_precomputed_digest_at_2g_and_high_memory() { let mut kernel = vec![0u8; 0x1000]; diff --git a/dstack/dstack-mr/src/lib.rs b/dstack/dstack-mr/src/lib.rs index 00385a05f..208b4412d 100644 --- a/dstack/dstack-mr/src/lib.rs +++ b/dstack/dstack-mr/src/lib.rs @@ -2,7 +2,6 @@ // // SPDX-License-Identifier: Apache-2.0 -use anyhow::{bail, Context, Result}; use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; @@ -24,167 +23,6 @@ mod tdvf; pub mod tdx; mod util; -/// Return the supported OVMF variant for a dstack OS version string ("MAJOR.MINOR.PATCH"). -/// -/// The version is still parsed for compatibility with callers that validate the -/// OS version through this helper, but all valid versions use `Pre202505`. -pub fn ovmf_variant_for_version(version: &str) -> Result<OvmfVariant> { - let parts: Vec<u32> = version - .split('.') - .map(|p| { - p.parse::<u32>() - .with_context(|| format!("invalid version component: {p}")) - }) - .collect::<Result<Vec<_>, _>>()?; - if parts.len() != 3 { - bail!("expected MAJOR.MINOR.PATCH, got {version}"); - } - Ok(OvmfVariant::Pre202505) -} - -/// Extract the `MAJOR.MINOR.PATCH` version suffix from a dstack image name. -/// -/// Recognises any `<prefix>-MAJOR.MINOR.PATCH[.SUFFIX]` shape, e.g. -/// `dstack-0.5.10`, `dstack-dev-0.5.10`, `dstack-nvidia-0.5.10`, -/// `dstack-nvidia-dev-0.5.10`, `dstack-0.5.10.rc1`, `dstack-dev-0.6.1.dev`. -/// -/// The optional `.SUFFIX` is permitted to be non-numeric (pre-release tag, -/// build label, etc.) and is dropped from the returned slice — only the -/// numeric `X.Y.Z` is needed to validate the image version. -/// -/// Returns `None` when the segment after the last `-` is not at least a valid -/// `X.Y.Z` triple of non-empty numeric components. -pub fn extract_version_from_image_name(image: &str) -> Option<&str> { - let tail = image.rsplit('-').next()?; - let parts: Vec<&str> = tail.split('.').collect(); - if !(3..=4).contains(&parts.len()) { - return None; - } - let core_numeric = parts[..3] - .iter() - .all(|p| !p.is_empty() && p.parse::<u32>().is_ok()); - let suffix_ok = parts.len() == 3 || !parts[3].is_empty(); - if !(core_numeric && suffix_ok) { - return None; - } - // Slice off the optional `.SUFFIX` so callers get just `X.Y.Z`. - let core_len = parts[0].len() + 1 + parts[1].len() + 1 + parts[2].len(); - Some(&tail[..core_len]) -} - -/// Return the supported OVMF variant from an image name like `dstack-0.5.10`. -/// -/// Falls back to `OvmfVariant::default()` (= `Pre202505`) when the image name is -/// missing or doesn't carry a parseable version suffix. Use this only as a -/// fallback for images that pre-date `VmConfig::ovmf_variant`. -pub fn ovmf_variant_for_image(image: Option<&str>) -> OvmfVariant { - image - .and_then(extract_version_from_image_name) - .and_then(|v| ovmf_variant_for_version(v).ok()) - .unwrap_or_default() -} - -#[cfg(test)] -mod ovmf_variant_tests { - use super::*; - - #[test] - fn pre_202505_for_all_versions() { - for v in [ - "0.4.99", "0.5.7", "0.5.8", "0.5.9", "0.5.10", "0.5.99", "0.6.0", "0.6.1", "0.6.2", - "0.7.0", "1.0.0", - ] { - assert_eq!( - ovmf_variant_for_version(v).unwrap(), - OvmfVariant::Pre202505, - "{v}" - ); - } - } - - #[test] - fn rejects_malformed_version() { - assert!(ovmf_variant_for_version("0.5").is_err()); - assert!(ovmf_variant_for_version("0.5.10-dev").is_err()); - assert!(ovmf_variant_for_version("v0.5.10").is_err()); - } - - #[test] - fn parses_version_from_image_name() { - // Three-segment plain versions across the known prefix shapes. - assert_eq!( - extract_version_from_image_name("dstack-0.5.10"), - Some("0.5.10") - ); - assert_eq!( - extract_version_from_image_name("dstack-dev-0.5.10"), - Some("0.5.10") - ); - assert_eq!( - extract_version_from_image_name("dstack-nvidia-0.5.10"), - Some("0.5.10") - ); - assert_eq!( - extract_version_from_image_name("dstack-nvidia-dev-0.6.1"), - Some("0.6.1") - ); - // Optional .SUFFIX is allowed and dropped; suffix may be non-numeric. - assert_eq!( - extract_version_from_image_name("dstack-0.5.10.rc1"), - Some("0.5.10") - ); - assert_eq!( - extract_version_from_image_name("dstack-dev-0.6.1.dev"), - Some("0.6.1") - ); - assert_eq!( - extract_version_from_image_name("dstack-nvidia-dev-0.5.10.1"), - Some("0.5.10") - ); - // Rejections. - assert_eq!(extract_version_from_image_name("dstack"), None); - assert_eq!(extract_version_from_image_name("dstack-rc1"), None); - assert_eq!(extract_version_from_image_name("dstack-0.5"), None); - assert_eq!(extract_version_from_image_name("dstack-0.5.10."), None); - assert_eq!(extract_version_from_image_name("dstack-0.5.10.1.2"), None); - assert_eq!(extract_version_from_image_name("dstack-0..10"), None); - assert_eq!(extract_version_from_image_name("dstack-a.b.c"), None); - } - - #[test] - fn ovmf_variant_for_image_handles_missing_and_unknown() { - assert_eq!(ovmf_variant_for_image(None), OvmfVariant::Pre202505); - assert_eq!( - ovmf_variant_for_image(Some("dstack")), - OvmfVariant::Pre202505 - ); - assert_eq!( - ovmf_variant_for_image(Some("dstack-0.5.9")), - OvmfVariant::Pre202505 - ); - assert_eq!( - ovmf_variant_for_image(Some("dstack-0.5.10")), - OvmfVariant::Pre202505 - ); - assert_eq!( - ovmf_variant_for_image(Some("dstack-nvidia-dev-0.6.1")), - OvmfVariant::Pre202505 - ); - } - - #[test] - fn serializes_with_snake_case() { - assert_eq!( - serde_json::to_string(&OvmfVariant::Pre202505).unwrap(), - "\"pre202505\"" - ); - assert_eq!( - serde_json::from_str::<OvmfVariant>("\"pre202505\"").unwrap(), - OvmfVariant::Pre202505 - ); - } -} - /// Contains all the measurement values for TDX. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TdxMeasurements { diff --git a/dstack/dstack-mr/src/machine.rs b/dstack/dstack-mr/src/machine.rs index 59d061084..7925d3df0 100644 --- a/dstack/dstack-mr/src/machine.rs +++ b/dstack/dstack-mr/src/machine.rs @@ -20,6 +20,12 @@ pub struct Machine<'a> { pub initrd: &'a str, pub kernel_cmdline: &'a str, pub two_pass_add_pages: Option<bool>, + /// Whether this image's OVMF normalizes the Linux setup header before + /// measuring the kernel. Defaults to `false`, which is the behavior of + /// every image built before the normalization landed; callers that have + /// the image metadata set it from `kernel_header_normalized`. + #[builder(default = false)] + pub normalized_setup_header: bool, pub pic: Option<bool>, pub qemu_version: Option<String>, #[builder(default = false)] @@ -33,6 +39,12 @@ pub struct Machine<'a> { /// predate this field keep the historical single-NIC layout. #[builder(default = 1)] pub num_nics: u32, + /// Number of virtio-blk verity volumes attached before the NICs. + #[builder(default)] + pub num_verity_volumes: u32, + /// Whether QEMU attaches a tpm-tis device backed by swtpm. + #[builder(default)] + pub swtpm: bool, pub hotplug_off: bool, pub root_verity: bool, #[builder(default)] @@ -44,17 +56,26 @@ pub struct Machine<'a> { } fn parse_version_tuple(v: &str) -> Result<(u32, u32, u32)> { - let parts: Vec<u32> = v - .split('.') - .map(|p| p.parse::<u32>().context("Invalid version number")) - .collect::<Result<Vec<_>, _>>()?; - if parts.len() != 3 { - bail!( - "Version string must have exactly 3 parts (major.minor.patch), got {}", - parts.len() - ); + let mut parts = v.split('.'); + let major = parts + .next() + .context("Version string must have exactly 3 parts (major.minor.patch)")? + .parse::<u32>() + .context("Invalid version number")?; + let minor = parts + .next() + .context("Version string must have exactly 3 parts (major.minor.patch)")? + .parse::<u32>() + .context("Invalid version number")?; + let patch = parts + .next() + .context("Version string must have exactly 3 parts (major.minor.patch)")? + .parse::<u32>() + .context("Invalid version number")?; + if parts.next().is_some() { + bail!("Version string must have exactly 3 parts (major.minor.patch)"); } - Ok((parts[0], parts[1], parts[2])) + Ok((major, minor, patch)) } impl Machine<'_> { @@ -122,6 +143,7 @@ impl Machine<'_> { initrd_data.len() as u32, self.memory_size, 0x28000, + self.normalized_setup_header, )?; debug_print_log("RTMR1", &rtmr1_log); let rtmr1 = measure_log(&rtmr1_log); diff --git a/dstack/dstack-mr/src/sev.rs b/dstack/dstack-mr/src/sev.rs index 157d083bf..b1d419a49 100644 --- a/dstack/dstack-mr/src/sev.rs +++ b/dstack/dstack-mr/src/sev.rs @@ -320,7 +320,17 @@ fn build_sev_hashes_page( Ok(page) } -fn measured_kernel_cmdline(input: &str) -> String { +/// Normalize an image-provided kernel command line for AMD SEV-SNP. +/// +/// Unlike TDX, SNP does **not** get OVMF's `initrd=initrd` suffix: with +/// `kernel-hashes=on` QEMU builds the SEV hash table itself from the exact +/// `-append` string instead of routing the initrd through OVMF's loader fs, so +/// the measured command line is the one the VMM passed. This therefore only +/// strips surrounding whitespace. +/// +/// Do not confuse it with [`crate::tdx::measured_kernel_cmdline`], which does +/// append the suffix. The two are not interchangeable. +fn normalize_kernel_cmdline(input: &str) -> String { input.trim().to_string() } @@ -674,7 +684,7 @@ pub fn compute_expected_measurement(input: &MeasurementInput) -> Result<[u8; 48] .as_deref() .ok_or_else(|| anyhow::anyhow!("vcpu_type is required"))?; - let cmdline = measured_kernel_cmdline(&input.base_cmdline); + let cmdline = normalize_kernel_cmdline(&input.base_cmdline); let resolved_sections = input .ovmf_sections .iter() @@ -746,10 +756,10 @@ fn sev_os_image_measurement( ) -> Result<dstack_types::SevOsImageMeasurement> { // Validate that the measured command line commits the rootfs identity. The // compact image projection does not carry a separate rootfs_hash because it - // is already committed by `kernel_cmdline_sha256`. + // is already committed by the `base_cmdline` string itself. rootfs_hash_from_cmdline(Some(&input.base_cmdline))?; Ok(dstack_types::SevOsImageMeasurement { - base_cmdline: measured_kernel_cmdline(&input.base_cmdline), + base_cmdline: normalize_kernel_cmdline(&input.base_cmdline), ovmf_hash: decode_required_hex("ovmf_hash", &input.ovmf_hash, 48)?, kernel_hash: decode_required_hex("kernel_hash", &input.kernel_hash, 32)?, initrd_hash: effective_initrd_hash_from_hex(&input.initrd_hash)?, @@ -862,11 +872,11 @@ pub fn sev_os_image_measurement_for_image_dir( let ovmf = ovmf_measurement_info(&image_dir.join(bios))?; // Validate that the measured command line commits the rootfs identity. The // compact image projection does not carry a separate rootfs_hash because it - // is already committed by `kernel_cmdline_sha256`. + // is already committed by the `base_cmdline` string itself. rootfs_hash_from_cmdline(meta.cmdline.as_deref())?; Ok(dstack_types::SevOsImageMeasurement { - base_cmdline: measured_kernel_cmdline( + base_cmdline: normalize_kernel_cmdline( meta.cmdline .as_deref() .context("metadata.json cmdline is required for amd sev-snp measurement")?, @@ -911,10 +921,15 @@ pub fn validate_mr_config(mr_config: &MrConfigV3) -> Result<()> { if mr_config.version != 3 { bail!("mr_config version must be 3"); } - ensure_len("mr_config.app_id", &mr_config.app_id, 20)?; + if let Some(app_id) = mr_config.app_id.as_deref() { + ensure_len("mr_config.app_id", app_id, 20)?; + } ensure_len("mr_config.compose_hash", &mr_config.compose_hash, 32)?; - if !mr_config.instance_id.is_empty() { - ensure_len("mr_config.instance_id", &mr_config.instance_id, 20)?; + if let Some(gpu_policy_hash) = &mr_config.gpu_policy_hash { + ensure_len("mr_config.gpu_policy_hash", gpu_policy_hash, 32)?; + } + if let Some(instance_id) = mr_config.instance_id.as_deref() { + ensure_len("mr_config.instance_id", instance_id, 20)?; } Ok(()) } @@ -1476,6 +1491,7 @@ mod tests { MrConfigV3::new( vec![0x11; 20], vec![0x22; 32], + None, dstack_types::KeyProviderKind::None, Vec::new(), vec![0x33; 20], @@ -1616,6 +1632,7 @@ mod tests { MrConfigV3::new( vec![0xee; 20], vec![0x22; 32], + None, dstack_types::KeyProviderKind::None, Vec::new(), vec![0x33; 20], @@ -1623,6 +1640,7 @@ mod tests { MrConfigV3::new( vec![0x11; 20], vec![0xee; 32], + None, dstack_types::KeyProviderKind::None, Vec::new(), vec![0x33; 20], @@ -1630,10 +1648,19 @@ mod tests { MrConfigV3::new( vec![0x11; 20], vec![0x22; 32], + None, dstack_types::KeyProviderKind::None, Vec::new(), vec![0xee; 20], ), + MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + Some(vec![0xee; 32]), + dstack_types::KeyProviderKind::None, + Vec::new(), + vec![0x33; 20], + ), ]; for evil in evil_mr_configs { let vm_config = synthetic_vm_config(&input, &evil); diff --git a/dstack/dstack-mr/src/tdx.rs b/dstack/dstack-mr/src/tdx.rs index abd6c7f99..72e29f405 100644 --- a/dstack/dstack-mr/src/tdx.rs +++ b/dstack/dstack-mr/src/tdx.rs @@ -11,8 +11,9 @@ //! intentionally excluded and must come from `VmConfig`. use crate::kernel::{ - patched_kernel_authenticode_sha384, tdx_kernel_hash_uses_precomputed_high_mem, - TDX_KERNEL_HASH_COMPAT_2G_MEMORY, TDX_KERNEL_HASH_STABLE_MIN_MEMORY, + kernel_authenticode_sha384, patched_kernel_authenticode_sha384, + tdx_kernel_hash_uses_precomputed_high_mem, TDX_KERNEL_HASH_COMPAT_2G_MEMORY, + TDX_KERNEL_HASH_STABLE_MIN_MEMORY, }; use crate::tdvf::{rtmr0_log_from_td_hob_hash_with_acpi_hashes, AcpiTableHashes, Tdvf}; use crate::util::{measure_log, measure_sha384}; @@ -33,9 +34,12 @@ struct ImageMetadata { initrd: String, bios: String, #[serde(default)] - version: String, - #[serde(default)] ovmf_variant: Option<OvmfVariant>, + /// Declares whether this image's OVMF normalizes the Linux setup header + /// before measuring the kernel. Absent on every image built before that + /// landed, and `false` is exactly their behavior. + #[serde(default)] + kernel_header_normalized: bool, } #[derive(Debug, Clone)] @@ -62,8 +66,14 @@ fn validate_bytes_field(value: &[u8], field: &str, expected_len: usize) -> Resul Ok(value.to_vec()) } -fn select_mrtd(measurement: &TdxOsImageMeasurement, vm_config: &VmConfig) -> Result<Vec<u8>> { - let machine = crate::Machine::builder() +/// Build the machine description the lite path measures against. +/// +/// Only the VM-shape inputs matter here: the firmware, kernel and initrd paths +/// stay empty because the lite path never reads image files, and the callers +/// (MRTD candidate selection and ACPI table generation) only consume the QEMU +/// topology knobs. +fn machine_from_vm_config(vm_config: &VmConfig, ovmf_variant: OvmfVariant) -> crate::Machine<'_> { + crate::Machine::builder() .cpu_count(vm_config.cpu_count) .memory_size(vm_config.memory_size) .firmware("") @@ -83,10 +93,40 @@ fn select_mrtd(measurement: &TdxOsImageMeasurement, vm_config: &VmConfig) -> Res .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) .num_nics(vm_config.num_nics) + .num_verity_volumes(vm_config.num_verity_volumes) + .swtpm(vm_config.swtpm) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) - .ovmf_variant(measurement.tdvf.ovmf_variant) - .build(); + .ovmf_variant(ovmf_variant) + .build() +} + +/// Recompute the three RTMR0 ACPI digests from the VM shape in `vm_config`. +/// +/// The ACPI tables QEMU hands to OVMF depend only on the deployment topology +/// (vCPU count, RAM size, PCI devices, QEMU version), never on the OS image, so +/// they can be regenerated without downloading anything. The lite path +/// otherwise replays the digests the guest reported in its event log, which +/// makes those three RTMR0 entries self-consistent but unconstrained; comparing +/// against these expected digests is what turns them into a verified value. +/// The verifier does exactly that on both TDX paths, and treats a mismatch -- +/// and a VM shape this cannot model -- as fatal rather than unverified. +pub fn expected_rtmr0_acpi_hashes( + vm_config: &VmConfig, + ovmf_variant: OvmfVariant, +) -> Result<TdxRtmr0AcpiHashes> { + let tables = machine_from_vm_config(vm_config, ovmf_variant) + .build_tables() + .context("failed to generate expected ACPI tables")?; + Ok(TdxRtmr0AcpiHashes { + loader: measure_sha384(&tables.loader), + rsdp: measure_sha384(&tables.rsdp), + tables: measure_sha384(&tables.tables), + }) +} + +fn select_mrtd(measurement: &TdxOsImageMeasurement, vm_config: &VmConfig) -> Result<Vec<u8>> { + let machine = machine_from_vm_config(vm_config, measurement.tdvf.ovmf_variant); let opts = machine .versioned_options() .context("failed to resolve QEMU measurement options")?; @@ -117,6 +157,12 @@ fn read_varuint(input: &mut &[u8]) -> Result<u64> { } } +/// q35 keeps guest RAM in one block below 4G until it reaches this size, then +/// caps the below-4G block at 2 GiB and moves the remainder above 4G +/// (`lowmem = 0xb0000000 unless ram_size >= 0xb0000000`). The TD HOB memory +/// ranges follow that split. +const Q35_HIGH_MEMORY_SPLIT: u64 = 0xB000_0000; + fn measure_td_hob_from_witness_data(data: &[u8], memory_size: u64) -> Result<Vec<u8>> { let mut input = data; let base_page = read_varuint(&mut input)?; @@ -183,7 +229,7 @@ fn measure_td_hob_from_witness_data(data: &[u8], memory_size: u64) -> Result<Vec if last_end < last_start { bail!("Invalid last memory range: end < start"); } - if memory_size >= TDX_KERNEL_HASH_STABLE_MIN_MEMORY { + if memory_size >= Q35_HIGH_MEMORY_SPLIT { if last_start < 0x80000000u64 { add_memory_resource_hob(0x07, last_start, 0x80000000u64 - last_start); } @@ -246,12 +292,22 @@ fn rtmr1_log_from_kernel_hash(kernel_hash: Vec<u8>) -> Vec<Vec<u8>> { ] } +/// The kernel command-line suffix OVMF appends before measuring. +/// +/// `QemuKernelLoaderFsDxe` exposes the initrd as a loader-fs file and appends +/// this so the kernel picks it up, so the measured command line is never the +/// bare image-provided one. Every site that reconstructs the measured command +/// line must go through [`measured_kernel_cmdline`] rather than restating this. +pub const OVMF_INITRD_CMDLINE_SUFFIX: &str = " initrd=initrd"; + /// Return the measured TDX kernel command line for a metadata cmdline. /// /// This mirrors the existing dstack TDX measurement replay path, which measures /// the image-provided cmdline plus OVMF/QEMU's `initrd=initrd` suffix. +/// +/// AMD SEV-SNP does not share this suffix; see `sev::normalize_kernel_cmdline`. pub fn measured_kernel_cmdline(base_cmdline: &str) -> String { - format!("{base_cmdline} initrd=initrd") + format!("{base_cmdline}{OVMF_INITRD_CMDLINE_SUFFIX}") } /// Generate the image-static TDX measurement material from an image directory. @@ -270,21 +326,12 @@ pub fn tdx_os_image_measurement_for_image_dir(image_dir: &Path) -> Result<TdxOsI // Validate that the image identity carried by the measured cmdline is // well-formed. The normalized rootfs hash is not stored separately to keep - // the TDX projection compact; it is already committed by the measured - // kernel command line digest. + // the TDX projection compact; it is already committed by the command line + // the document carries. crate::sev::rootfs_hash_from_cmdline(Some(&base_cmdline)) .context("failed to parse dstack.rootfs_hash from TDX cmdline")?; - let ovmf_variant = meta - .ovmf_variant - .or_else(|| { - if meta.version.is_empty() { - None - } else { - crate::ovmf_variant_for_version(&meta.version).ok() - } - }) - .unwrap_or_default(); + let ovmf_variant = meta.ovmf_variant.unwrap_or_default(); let fw_data = fs::read(image_dir.join(&meta.bios)) .with_context(|| format!("cannot read {}", image_dir.join(&meta.bios).display()))?; @@ -296,19 +343,24 @@ pub fn tdx_os_image_measurement_for_image_dir(image_dir: &Path) -> Result<TdxOsI let kernel_path = image_dir.join(&meta.kernel); let kernel = fs::read(&kernel_path).with_context(|| format!("cannot read {}", kernel_path.display()))?; - let kernel_authenticode = patched_kernel_authenticode_sha384( - &kernel, - initrd.len() as u32, - TDX_KERNEL_HASH_STABLE_MIN_MEMORY, - 0x28000, - ) - .context("failed to compute high-memory QEMU-patched kernel hash")?; + // Which bytes OVMF will measure is a property of this image's firmware, so + // it is read from the image rather than from anything the host says. + let kernel_authenticode = if meta.kernel_header_normalized { + kernel_authenticode_sha384(&kernel)? + } else { + patched_kernel_authenticode_sha384( + &kernel, + initrd.len() as u32, + TDX_KERNEL_HASH_STABLE_MIN_MEMORY, + 0x28000, + ) + .context("failed to compute high-memory QEMU-patched kernel hash")? + }; Ok(TdxOsImageMeasurement { + kernel_header_normalized: meta.kernel_header_normalized, image: TdxImageMeasurement { - kernel_cmdline_sha384: crate::kernel::measure_cmdline(&measured_kernel_cmdline( - &base_cmdline, - )), + base_cmdline: base_cmdline.clone(), kernel_authenticode, initrd_sha384: measure_sha384(&initrd), }, @@ -336,19 +388,23 @@ pub fn tdx_measurement_hash_for_image_dir(image_dir: &Path) -> Result<[u8; 32]> /// Compute expected TDX measurements from self-contained TDX measurement /// material and the three ACPI table digests captured in RTMR[0]. /// -/// This path intentionally does not download or read the OS image. Because -/// QEMU's patched kernel Authenticode hash depends on exact guest RAM below -/// `TDX_KERNEL_HASH_STABLE_MIN_MEMORY`, the no-image-download path supports -/// CVMs at or above that threshold plus the exact 2 GiB placement, which QEMU -/// patches to the same kernel bytes as the high-memory case. +/// This path intentionally does not download or read the OS image. Every +/// guest memory size is supported: the kernel digest is a plain hash of the +/// image file, because the setup header is normalized on both sides. pub fn tdx_measurements_from_measurement_document( document: &TdxOsImageMeasurementDocument, vm_config: &VmConfig, acpi_hashes: &TdxRtmr0AcpiHashes, ) -> Result<crate::TdxMeasurements> { - if !tdx_kernel_hash_uses_precomputed_high_mem(vm_config.memory_size) { + let measurement = document + .decode_measurement() + .map_err(anyhow::Error::msg) + .context("failed to decode TDX measurement CBOR")?; + if !measurement.kernel_header_normalized + && !tdx_kernel_hash_uses_precomputed_high_mem(vm_config.memory_size) + { bail!( - "TDX lite attestation without image download requires memory_size == {} bytes ({} MiB) or >= {} bytes ({} MiB); got {} bytes", + "TDX lite attestation without image download requires memory_size == {} bytes ({} MiB) or >= {} bytes ({} MiB); got {} bytes. This restriction only applies to images whose OVMF does not normalize the kernel setup header, because QEMU's rewrite moves the initrd with guest RAM; re-emit the image to remove it", TDX_KERNEL_HASH_COMPAT_2G_MEMORY, TDX_KERNEL_HASH_COMPAT_2G_MEMORY / 1024 / 1024, TDX_KERNEL_HASH_STABLE_MIN_MEMORY, @@ -356,11 +412,6 @@ pub fn tdx_measurements_from_measurement_document( vm_config.memory_size ); } - - let measurement = document - .decode_measurement() - .map_err(anyhow::Error::msg) - .context("failed to decode TDX measurement CBOR")?; let mrtd = select_mrtd(&measurement, vm_config)?; let td_hob_hash = @@ -390,11 +441,13 @@ pub fn tdx_measurements_from_measurement_document( "tdx.measurement.image.initrd_sha384", 48, )?; - let kernel_cmdline_hash = validate_bytes_field( - &measurement.image.kernel_cmdline_sha384, - "tdx.measurement.image.kernel_cmdline_sha384", - 48, - )?; + // Now that the document carries the command line rather than its digest, + // the no-download path can enforce the same rootfs-identity invariant the + // build path does, instead of trusting an opaque 48-byte value. + crate::sev::rootfs_hash_from_cmdline(Some(&measurement.image.base_cmdline)) + .context("failed to parse dstack.rootfs_hash from tdx.measurement.image.base_cmdline")?; + let kernel_cmdline_hash = + crate::kernel::measure_cmdline(&measured_kernel_cmdline(&measurement.image.base_cmdline)); let rtmr2 = measure_log(&[kernel_cmdline_hash, initrd_hash]); Ok(crate::TdxMeasurements { @@ -442,14 +495,7 @@ pub fn tdx_measurements_for_image_dir_without_rtmr0( let ovmf_variant = vm_config .ovmf_variant .or(meta.ovmf_variant) - .or_else(|| { - if meta.version.is_empty() { - None - } else { - crate::ovmf_variant_for_version(&meta.version).ok() - } - }) - .unwrap_or_else(|| crate::ovmf_variant_for_image(vm_config.image.as_deref())); + .unwrap_or_default(); let firmware = firmware_path.display().to_string(); let kernel = kernel_path.display().to_string(); @@ -464,6 +510,7 @@ pub fn tdx_measurements_for_image_dir_without_rtmr0( .root_verity(true) .hotplug_off(vm_config.hotplug_off) .maybe_two_pass_add_pages(vm_config.qemu_single_pass_add_pages) + .normalized_setup_header(meta.kernel_header_normalized) .maybe_pic(vm_config.pic) .maybe_qemu_version(vm_config.qemu_version.clone()) .maybe_pci_hole64_size(if vm_config.pci_hole64_size > 0 { @@ -474,6 +521,8 @@ pub fn tdx_measurements_for_image_dir_without_rtmr0( .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) .num_nics(vm_config.num_nics) + .num_verity_volumes(vm_config.num_verity_volumes) + .swtpm(vm_config.swtpm) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(ovmf_variant) @@ -487,6 +536,7 @@ pub fn tdx_measurements_for_image_dir_without_rtmr0( initrd_data.len() as u32, vm_config.memory_size, 0x28000, + meta.kernel_header_normalized, ) .context("failed to compute RTMR1")?; let rtmr1 = measure_log(&rtmr1_log); @@ -538,14 +588,7 @@ pub fn tdx_measurements_for_image_dir_with_acpi_hashes( let ovmf_variant = vm_config .ovmf_variant .or(meta.ovmf_variant) - .or_else(|| { - if meta.version.is_empty() { - None - } else { - crate::ovmf_variant_for_version(&meta.version).ok() - } - }) - .unwrap_or_else(|| crate::ovmf_variant_for_image(vm_config.image.as_deref())); + .unwrap_or_default(); let firmware = firmware_path.display().to_string(); let kernel = kernel_path.display().to_string(); @@ -560,6 +603,7 @@ pub fn tdx_measurements_for_image_dir_with_acpi_hashes( .root_verity(true) .hotplug_off(vm_config.hotplug_off) .maybe_two_pass_add_pages(vm_config.qemu_single_pass_add_pages) + .normalized_setup_header(meta.kernel_header_normalized) .maybe_pic(vm_config.pic) .maybe_qemu_version(vm_config.qemu_version.clone()) .maybe_pci_hole64_size(if vm_config.pci_hole64_size > 0 { @@ -570,6 +614,8 @@ pub fn tdx_measurements_for_image_dir_with_acpi_hashes( .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) .num_nics(vm_config.num_nics) + .num_verity_volumes(vm_config.num_verity_volumes) + .swtpm(vm_config.swtpm) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(ovmf_variant) @@ -596,6 +642,7 @@ pub fn tdx_measurements_for_image_dir_with_acpi_hashes( initrd_data.len() as u32, vm_config.memory_size, 0x28000, + meta.kernel_header_normalized, ) .context("failed to compute RTMR1")?; let rtmr1 = measure_log(&rtmr1_log); @@ -613,3 +660,75 @@ pub fn tdx_measurements_for_image_dir_with_acpi_hashes( rtmr2, }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::kernel::measure_cmdline; + + /// Shaped like a real `os/image/kernel-cmdline.sh` command line: it always + /// carries `dstack.rootfs_hash`, which is what pins the rootfs. + const SAMPLE_BASE_CMDLINE: &str = "console=ttyS0 init=/init panic=1 \ + dstack.rootfs_hash=1111111111111111111111111111111111111111111111111111111111111111 \ + dstack.rootfs_size=4096"; + + fn sample_measurement() -> TdxOsImageMeasurement { + TdxOsImageMeasurement { + image: TdxImageMeasurement { + base_cmdline: SAMPLE_BASE_CMDLINE.to_string(), + kernel_authenticode: vec![0x22; 48], + initrd_sha384: vec![0x33; 48], + }, + tdvf: TdxTdvfMeasurement { + ovmf_variant: OvmfVariant::Pre202505, + mrtd: TdxMrtdCandidates { + single_pass: vec![0x44; 48], + two_pass: vec![0x55; 48], + }, + td_hob_witness: vec![0x66; 16], + }, + kernel_header_normalized: true, + } + } + + #[test] + fn measured_kernel_cmdline_appends_the_ovmf_suffix() { + assert_eq!( + measured_kernel_cmdline(SAMPLE_BASE_CMDLINE), + format!("{SAMPLE_BASE_CMDLINE}{OVMF_INITRD_CMDLINE_SUFFIX}"), + ); + assert_eq!(OVMF_INITRD_CMDLINE_SUFFIX, " initrd=initrd"); + } + + /// Golden vector. The command-line event is the first RTMR[2] entry, so a + /// change here silently invalidates every deployed `os_image_hash`. If this + /// fails, the measurement protocol changed: that must be a deliberate, + /// reviewed decision, not a side effect. + #[test] + fn rtmr2_command_line_event_digest_is_stable() { + let digest = measure_cmdline(&measured_kernel_cmdline(SAMPLE_BASE_CMDLINE)); + assert_eq!(hex::encode(digest), "bb4154e6e429e184bc63d544ae5720f868e5859b378b13bab69860e1fc65c09f1b67277bba010841fbe8bc3619e58d58"); + } + + /// Golden vector for the full RTMR[2] replay (command line, then initrd). + #[test] + fn rtmr2_replay_is_stable() { + let cmdline_digest = measure_cmdline(&measured_kernel_cmdline(SAMPLE_BASE_CMDLINE)); + let initrd_digest = vec![0x33; 48]; + assert_eq!( + hex::encode(measure_log(&[cmdline_digest, initrd_digest])), + "2fb6d31492cd2f073fe8fdaa9dbdff4dfdde92bf07f9a16c2b7123ae7be5090007974d33bfe8807ac56974160f8f2948", + ); + } + + /// Golden vector for the CBOR that `sha256sum.txt` commits to, and hence + /// for `os_image_hash` itself. Changing the encoding requires rebuilding and + /// re-registering every image, so it must never change accidentally. + #[test] + fn tdx_measurement_document_cbor_is_stable() { + assert_eq!( + hex::encode(sample_measurement().to_cbor_vec()), + "a36776657273696f6e0465696d616765a467636d646c696e65788c636f6e736f6c653d747479533020696e69743d2f696e69742070616e69633d312064737461636b2e726f6f7466735f686173683d313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131312064737461636b2e726f6f7466735f73697a653d34303936736b65726e656c5f61757468656e7469636f6465583022222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222278186b65726e656c5f6865616465725f6e6f726d616c697a6564f56d696e697472645f73686133383458303333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333336474647666a3646f766d6669707265323032353035646d727464a26b73696e676c655f7061737358304444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444446874776f5f7061737358305555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555556674645f686f625066666666666666666666666666666666", + ); + } +} diff --git a/dstack/dstack-mr/tests/tdvf_parse.rs b/dstack/dstack-mr/tests/tdvf_parse.rs index 6c7e93827..ed0bbecad 100644 --- a/dstack/dstack-mr/tests/tdvf_parse.rs +++ b/dstack/dstack-mr/tests/tdvf_parse.rs @@ -88,7 +88,7 @@ fn test_tdvf_parse_produces_correct_measurements() -> Result<()> { let firmware_path = image_dir.join(&image_info.bios).display().to_string(); let kernel_path = image_dir.join(&image_info.kernel).display().to_string(); let initrd_path = image_dir.join(&image_info.initrd).display().to_string(); - let cmdline = image_info.cmdline + " initrd=initrd"; + let cmdline = dstack_mr::tdx::measured_kernel_cmdline(&image_info.cmdline); eprintln!("Building machine configuration..."); let machine = Machine::builder() diff --git a/dstack/dstack-types/Cargo.toml b/dstack/dstack-types/Cargo.toml index 526d5192b..5dba0358c 100644 --- a/dstack/dstack-types/Cargo.toml +++ b/dstack/dstack-types/Cargo.toml @@ -16,6 +16,7 @@ or-panic.workspace = true scale = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] } serde-human-bytes.workspace = true +serde_with.workspace = true serde_jcs.workspace = true serde_json.workspace = true sha2.workspace = true diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index a6fe1901f..944bf12e1 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -10,6 +10,89 @@ use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; use size_parser::human_size; +/// Bound event-log growth and MrConfigV3 size while supporting independent +/// infrastructure-provider initialization stages. +pub const MAX_INIT_SCRIPTS: usize = 5; + +pub mod init_script_hashes { + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use serde_human_bytes::ByteBuf; + + pub fn serialize<S>(values: &[Vec<u8>], serializer: S) -> Result<S::Ok, S::Error> + where + S: Serializer, + { + values + .iter() + .cloned() + .map(ByteBuf::from) + .collect::<Vec<_>>() + .serialize(serializer) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<Vec<u8>>, D::Error> + where + D: Deserializer<'de>, + { + let values = Vec::<ByteBuf>::deserialize(deserializer)?; + if values.len() > super::MAX_INIT_SCRIPTS { + return Err(serde::de::Error::custom(format!( + "init_script_hashes supports at most {} hashes", + super::MAX_INIT_SCRIPTS + ))); + } + if values.iter().any(|value| value.len() != 32) { + return Err(serde::de::Error::custom( + "each init_script_hash must be 32 bytes", + )); + } + Ok(values.into_iter().map(ByteBuf::into_vec).collect()) + } + + pub mod option { + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use serde_human_bytes::ByteBuf; + + pub fn serialize<S>(values: &Option<Vec<Vec<u8>>>, serializer: S) -> Result<S::Ok, S::Error> + where + S: Serializer, + { + values + .as_ref() + .map(|values| { + values + .iter() + .cloned() + .map(ByteBuf::from) + .collect::<Vec<_>>() + }) + .serialize(serializer) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<Vec<Vec<u8>>>, D::Error> + where + D: Deserializer<'de>, + { + Option::<Vec<ByteBuf>>::deserialize(deserializer)? + .map(|values| { + if values.len() > super::super::MAX_INIT_SCRIPTS { + return Err(serde::de::Error::custom(format!( + "init_script_hashes supports at most {} hashes", + super::super::MAX_INIT_SCRIPTS + ))); + } + if values.iter().any(|value| value.len() != 32) { + return Err(serde::de::Error::custom( + "each init_script_hash must be 32 bytes", + )); + } + Ok(values.into_iter().map(ByteBuf::into_vec).collect()) + }) + .transpose() + } + } +} + /// Identifies which OVMF flavour the guest image was built with. /// /// Only the pre-202505 OVMF measurement layout is supported. @@ -71,6 +154,51 @@ impl TdxAttestationVariant { } } +/// Event log version controlling the digest format. +/// +/// Using an enum ensures exhaustive matching — adding a new version +/// forces all match sites to be updated. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum EventLogVersion { + /// Legacy binary digest: `SHA384(event_type_le || ":" || name || ":" || payload)` + #[default] + V1, + /// JSON canonical digest (JCS RFC 8785), hashed as canonical JSON bytes: + /// `SHA384({"name":"...","payload":"hex...","type":134217729})` + V2, +} + +impl EventLogVersion { + pub fn is_v1(&self) -> bool { + matches!(self, Self::V1) + } + + pub fn from_u32(v: u32) -> Option<Self> { + match v { + 1 => Some(EventLogVersion::V1), + 2 => Some(EventLogVersion::V2), + _ => None, + } + } +} + +impl Serialize for EventLogVersion { + fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> { + match self { + EventLogVersion::V1 => serializer.serialize_u32(1), + EventLogVersion::V2 => serializer.serialize_u32(2), + } + } +} + +impl<'de> Deserialize<'de> for EventLogVersion { + fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> { + let v = u32::deserialize(deserializer)?; + EventLogVersion::from_u32(v) + .ok_or_else(|| serde::de::Error::custom(format!("unknown event log version: {v}"))) + } +} + #[derive(Deserialize, Serialize, Debug, Clone)] pub struct AppCompose { #[serde(deserialize_with = "deserialize_manifest_version")] @@ -80,8 +208,23 @@ pub struct AppCompose { #[serde(default)] pub features: Vec<String>, pub runner: String, + /// containerd snapshotter used by the `nerdctl-compose` runner. + /// The field is invalid for other runners. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub snapshotter: Option<ContainerSnapshotter>, #[serde(default)] pub docker_compose_file: Option<String>, + /// Bash scripts executed before the application runner starts. + /// + /// A single string is accepted for backward compatibility and is treated + /// as a one-element list. + #[serde( + default, + deserialize_with = "deserialize_init_scripts", + serialize_with = "serialize_init_scripts", + skip_serializing_if = "Vec::is_empty" + )] + pub init_script: Vec<String>, #[serde(default)] pub public_logs: bool, #[serde(default)] @@ -90,7 +233,11 @@ pub struct AppCompose { pub public_tcbinfo: bool, #[serde(default)] pub kms_enabled: bool, - #[serde(deserialize_with = "deserialize_gateway_enabled", flatten)] + #[serde( + deserialize_with = "deserialize_gateway_enabled", + serialize_with = "serialize_gateway_enabled", + flatten + )] pub gateway_enabled: bool, #[serde(default)] pub local_key_provider_enabled: bool, @@ -106,8 +253,14 @@ pub struct AppCompose { pub secure_time: bool, #[serde(default)] pub storage_fs: Option<String>, + /// Return unused data-disk blocks to the host. Disable this when leaking + /// filesystem allocation and deletion patterns is unacceptable. + #[serde(default = "default_true")] + pub storage_discard: bool, #[serde(default, with = "human_size")] pub swap_size: u64, + #[serde(default, skip_serializing_if = "EventLogVersion::is_v1")] + pub event_log_version: EventLogVersion, /// Per-port policy consumed by the gateway (PROXY protocol opt-in, /// optional port whitelist). #[serde(default)] @@ -119,15 +272,186 @@ pub struct AppCompose { /// of silently ignoring the requirements. #[serde(default, skip_serializing_if = "Option::is_none")] pub requirements: Option<Requirements>, + /// Read-only, dm-verity-protected volumes pre-seeded into the CVM. Each + /// `verity_root` is measured (it is part of these compose bytes), so the + /// guest only mounts content matching the attested app. See + /// docs/verity-volumes.md. + #[serde(default)] + pub verity_volumes: Vec<VerityVolume>, +} + +fn deserialize_init_scripts<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error> +where + D: serde::Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum InitScripts { + One(String), + Many(Vec<String>), + } + + let scripts = match Option::<InitScripts>::deserialize(deserializer)? { + None => Vec::new(), + Some(InitScripts::One(script)) => vec![script], + Some(InitScripts::Many(scripts)) => scripts, + }; + if scripts.len() > MAX_INIT_SCRIPTS { + return Err(serde::de::Error::custom(format!( + "init_script supports at most {MAX_INIT_SCRIPTS} scripts" + ))); + } + Ok(scripts) +} + +fn serialize_init_scripts<S>(scripts: &[String], serializer: S) -> Result<S::Ok, S::Error> +where + S: serde::Serializer, +{ + if let [script] = scripts { + serializer.serialize_str(script) + } else { + scripts.serialize(serializer) + } +} + +/// A pre-baked, read-only dm-verity volume attached to the CVM. +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct VerityVolume { + /// Bare image file name resolved by the VMM under `cvm.volumes_dir`. + pub source: String, + /// dm-verity root hash (hex): the volume's content identity and integrity + /// check. The guest matches attached devices against it. + #[serde(with = "hex_bytes")] + pub verity_root: [u8; 32], + /// Absolute path where the volume's filesystem is mounted. + #[serde(deserialize_with = "deserialize_absolute_path")] + pub target: std::path::PathBuf, +} + +/// Reject ambiguous mount declarations before any disk is attached or +/// activated. The same root may intentionally be mounted at multiple targets, +/// but a target can only be owned by one volume. +pub fn validate_verity_volumes(volumes: &[VerityVolume]) -> Result<(), String> { + let mut targets = std::collections::HashSet::new(); + for volume in volumes { + if !targets.insert(&volume.target) { + return Err(format!( + "duplicate verity volume target {}", + volume.target.display() + )); + } + } + Ok(()) +} + +fn deserialize_absolute_path<'de, D>(deserializer: D) -> Result<std::path::PathBuf, D::Error> +where + D: serde::Deserializer<'de>, +{ + let value = String::deserialize(deserializer)?; + let path = std::path::PathBuf::from(&value); + if !path.is_absolute() { + return Err(serde::de::Error::custom(format!( + "volume target must be an absolute path, got '{value}'" + ))); + } + Ok(path) +} + +#[cfg(test)] +mod verity_volume_tests { + use super::{validate_verity_volumes, VerityVolume}; + + fn volume(root: u8, target: &str) -> VerityVolume { + VerityVolume { + source: format!("{root}.img"), + verity_root: [root; 32], + target: target.into(), + } + } + + #[test] + fn allows_duplicate_roots_but_rejects_duplicate_targets() { + validate_verity_volumes(&[volume(1, "/a"), volume(1, "/b")]).unwrap(); + assert!(validate_verity_volumes(&[volume(1, "/a"), volume(2, "/a")]) + .unwrap_err() + .contains("duplicate verity volume target")); + validate_verity_volumes(&[volume(1, "/a"), volume(2, "/b")]).unwrap(); + } +} + +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum ContainerSnapshotter { + Overlayfs, + Stargz, +} + +/// Canonical source for the policy used when `requirements.gpu_policy` is +/// absent. Both typed defaults and measurement are derived from this JSON. +pub const DEFAULT_GPU_POLICY: &str = "{}"; + +/// Path containing the complete output of the NVIDIA GPU attestation command. +pub const GPU_ATTESTATION_OUTPUT: &str = "/run/nvidia-gpu-attestation/attestation.out"; + +/// Computes the SHA-256 digest of the JCS-canonicalized raw +/// `requirements.gpu_policy` JSON value. An absent policy is equivalent to +/// the default empty object. +pub fn gpu_policy_hash(compose_json: &[u8]) -> Result<[u8; 32], serde_json::Error> { + use sha2::{Digest, Sha256}; + + let compose: serde_json::Value = serde_json::from_slice(compose_json)?; + let default_policy: serde_json::Value = serde_json::from_str(DEFAULT_GPU_POLICY)?; + let policy = compose + .pointer("/requirements/gpu_policy") + .unwrap_or(&default_policy); + let canonical = serde_jcs::to_vec(policy)?; + Ok(Sha256::digest(canonical).into()) +} + +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct GpuPolicy { + /// Whether an attached GPU must pass local TEE attestation before the + /// guest continues booting. Defaults to true. + #[serde(default = "default_true")] + pub attest_gpu: bool, + /// Optional Rego v0 policy evaluated against NVIDIA nvattest's `claims` + /// array. It must define the boolean rule `data.policy.nv_match`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rego: Option<String>, + /// Permit NVIDIA DevTools mode. This defaults to false because DevTools + /// disables the GPU memory-confidentiality guarantees expected in + /// production. + #[serde(default)] + pub allow_devtools: bool, + /// Permit claims whose GPU attestation debug status is `enabled`. Defaults + /// to false. + #[serde(default)] + pub allow_debug: bool, + /// Permit claims that do not assert GPU secure boot. Defaults to false. + #[serde(default)] + pub allow_insecure_boot: bool, +} + +impl Default for GpuPolicy { + fn default() -> Self { + serde_json::from_str(DEFAULT_GPU_POLICY) + .or_panic("DEFAULT_GPU_POLICY must be a valid GPU policy") + } +} + +impl GpuPolicy { + /// Returns true when no application-specific GPU policy setting is set. + pub fn is_default(&self) -> bool { + self == &Self::default() + } } #[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq, Eq)] #[serde(default, deny_unknown_fields)] pub struct Requirements { - /// OS-version requirement parsed with Rust semver requirement semantics, - /// e.g. `">=0.6.0"` or `">=0.6.0, <0.7.0"`. - #[serde(skip_serializing_if = "Option::is_none")] - pub os_version: Option<String>, /// Allowed attestation platforms. Omitted means any supported platform; /// an explicit empty list means no platform is allowed. #[serde(skip_serializing_if = "Option::is_none")] @@ -149,28 +473,142 @@ pub struct Requirements { /// (e.g. 32 random alphanumeric characters). #[serde(skip_serializing_if = "Option::is_none")] pub launch_token_hash: Option<String>, - /// GPU TEE attestation requirement, defaults to `true` when omitted. + /// Application GPU policy applied before key provisioning. An omitted + /// field is parsed and measured as the default empty policy `{}`. /// - /// On guests with an NVIDIA GPU attached, `true` means the guest runs - /// local GPU attestation (nvattest) during system setup and refuses to - /// boot — before key provisioning — if the GPU fails to attest (e.g. a - /// non-CC GPU or CC mode disabled by the host). `false` skips attestation - /// and sets the GPU ready state directly. Guests without a GPU attached - /// are unaffected either way. - #[serde(skip_serializing_if = "Option::is_none")] - pub attest_gpu: Option<bool>, + /// Its original JSON value is JCS-canonicalized and its SHA-256 digest is + /// emitted as the `gpu-policy-hash` launch event immediately after + /// `compose-hash`. When the field is absent, `{}` is measured. An explicitly + /// present default-valued field remains part of the raw measurement. + /// Rego receives an empty claims array when no GPU attestation is produced, + /// allowing applications to enforce an expected GPU count. + #[serde(default, skip_serializing_if = "GpuPolicy::is_default")] + pub gpu_policy: GpuPolicy, + /// Whether the gateway should gate this app's traffic on its health, i.e. + /// hold an instance out of its app's load-balancing rotation until the + /// guest agent reports that the app is serving. + /// + /// Opt-in, and deliberately placed under `requirements` rather than at the + /// top level of `app-compose.json`: `Requirements` is `deny_unknown_fields` + /// and is itself gated behind `manifest_version >= 3`, so a deployment that + /// asks for health gating cannot land on a guest image that would silently + /// ignore it. An app that never sets this registers as "do not poll me" and + /// is routed to exactly as it was before this feature existed. + /// + /// Two flat fields rather than one nested object, and not because flat is + /// prettier. Every SDK has to reproduce this structure byte for byte to + /// compute the compose hash that gets whitelisted on chain, and a nested + /// object is where they drift: Go's `HealthCheck` had no passthrough for + /// fields it did not know and would have hashed them away silently, while + /// Python's `Requirements.from_dict` did not reconstruct the object at all + /// and raised on any round trip. Scalars under a `deny_unknown_fields` + /// parent have neither failure mode. + #[serde(default, skip_serializing_if = "is_false")] + pub health_check: bool, + /// Absolute path to a file the app writes its own verdict into. + /// + /// Two lines, in order: + /// + /// ```text + /// healthy + /// 1771234567 + /// ``` + /// + /// Line 1 is `healthy` or `unhealthy` (case-insensitive). Line 2 is the + /// unix timestamp, in seconds, at which the app wrote the file. The + /// timestamp is not decoration: a file older than + /// [`HEALTH_FILE_MAX_AGE_SECS`] counts as unhealthy, which is what turns a + /// wedged app -- still running, no longer updating anything -- into a + /// verdict instead of a stale `healthy` that never expires. Refresh it at + /// least twice per that window. + /// + /// When this is absent the agent falls back to the container runtime: + /// every container that declares a Compose `healthcheck` must be running + /// and healthy. That default needs nothing from the app, but it can only + /// see what the runtime sees; a file gives the app the last word. + /// + /// `Option`, not `String`. An empty path and an absent one have to stay + /// distinguishable, because Go's `omitempty` cannot tell them apart and + /// would hash `""` differently from the other SDKs -- on a digest that goes + /// on chain. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub health_status_file: Option<String>, } impl Requirements { pub fn is_empty(&self) -> bool { - self.os_version.is_none() - && self.platforms.is_none() + self.platforms.is_none() && self.tdx_measure_acpi_tables.is_none() && self.launch_token_hash.is_none() - && self.attest_gpu.is_none() + && self.gpu_policy.is_default() + && !self.health_check + && self.health_status_file.is_none() } } +/// Make a string that came from an untrusted party safe to put in a log line, +/// and bound its length. +/// +/// Both ends of the health path need this and neither can rely on the other: +/// the guest agent sanitizes what an app wrote before reporting it, and the +/// gateway sanitizes what a guest agent reported before logging it, because a +/// guest agent is exactly the party the gateway does not trust. +/// +/// "Unsafe" is wider than [`char::is_control`], which is only Unicode category +/// Cc. It misses two families that do the same damage: +/// +/// - `U+2028` LINE SEPARATOR and `U+2029` PARAGRAPH SEPARATOR, which many log +/// viewers -- and every JavaScript or JSON consumer downstream of one -- +/// treat as a line break. That is the log-forging that stripping `\n` was +/// meant to prevent. +/// - The bidirectional formatting characters (`U+202A`..`U+202E`, +/// `U+2066`..`U+2069`, `U+200E`, `U+200F`) and `U+FEFF`. `U+202E` reverses +/// the rendering of everything after it, so an attacker controls how the +/// rest of the line reads in a terminal without controlling its bytes. +/// +/// Truncation is by bytes, and it says so when it happens: a bound that +/// silently cuts a reason is worse than a short one, because the reader cannot +/// tell a complete message from a clipped one. +pub fn sanitize_for_log(text: &str, max_bytes: usize) -> String { + const MARKER: &str = "... (truncated)"; + let mut cleaned = String::with_capacity(text.len().min(max_bytes)); + let mut truncated = false; + for ch in text.chars() { + let ch = if is_unsafe_to_log(ch) { ' ' } else { ch }; + if cleaned.len() + ch.len_utf8() > max_bytes { + truncated = true; + break; + } + cleaned.push(ch); + } + if truncated { + cleaned.push_str(MARKER); + } + cleaned +} + +/// Whether a character must not survive into a log line. See +/// [`sanitize_for_log`]. +fn is_unsafe_to_log(ch: char) -> bool { + ch.is_control() + || matches!( + ch, + '\u{2028}' + | '\u{2029}' + | '\u{200E}' + | '\u{200F}' + | '\u{202A}'..='\u{202E}' + | '\u{2066}'..='\u{2069}' + | '\u{FEFF}' + ) +} + +/// How old a `health_status_file` may be before it is read as unhealthy. +/// +/// Fixed rather than configurable: this is a liveness bound, and an app that +/// wants a longer one is asking for its own failures to take longer to notice. +pub const HEALTH_FILE_MAX_AGE_SECS: u64 = 60; + /// Domain-separation prefix for [`launch_token_hash`]. It keeps the digest /// distinct from a plain `sha256(token)` (as used by the legacy app-layer /// top-level `launch_token_hash` convention) and from generic precomputed @@ -298,6 +736,21 @@ where Ok(value.gateway_enabled || value.tproxy_enabled) } +fn serialize_gateway_enabled<S>(enabled: &bool, serializer: S) -> Result<S::Ok, S::Error> +where + S: serde::Serializer, +{ + #[derive(Serialize)] + struct GatewayEnabled { + gateway_enabled: bool, + } + + GatewayEnabled { + gateway_enabled: *enabled, + } + .serialize(serializer) +} + #[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum KeyProviderKind { @@ -362,24 +815,6 @@ impl AppCompose { } } } - - /// Whether an attached GPU must pass local TEE attestation before the - /// guest continues booting. Defaults to `true` when - /// `requirements.attest_gpu` is omitted. - /// - /// `requirements` are only valid on manifest_version >= 3 (guests reject - /// older manifests carrying them); the opt-out is additionally ignored on - /// legacy manifests here so a caller that skipped that validation still - /// fails closed. - pub fn attest_gpu(&self) -> bool { - if !matches!(self.manifest_version_u32(), Some(v) if v >= 3) { - return true; - } - self.requirements - .as_ref() - .and_then(|r| r.attest_gpu) - .unwrap_or(true) - } } #[cfg(test)] @@ -394,6 +829,85 @@ mod app_compose_tests { })) } + #[test] + fn storage_discard_defaults_on_and_can_be_disabled() { + assert!(parse_compose(serde_json::json!(2)).unwrap().storage_discard); + + let compose: AppCompose = serde_json::from_value(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "storage_discard": false + })) + .unwrap(); + assert!(!compose.storage_discard); + } + + #[test] + fn init_script_accepts_string_array_and_null() { + assert!(parse_compose(serde_json::json!(2)) + .unwrap() + .init_script + .is_empty()); + + let single: AppCompose = serde_json::from_value(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "init_script": "echo one" + })) + .unwrap(); + assert_eq!(single.init_script, ["echo one"]); + assert_eq!( + serde_json::to_value(&single).unwrap()["init_script"], + "echo one" + ); + + let multiple: AppCompose = serde_json::from_value(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "init_script": ["echo one", "echo two"] + })) + .unwrap(); + assert_eq!(multiple.init_script, ["echo one", "echo two"]); + assert_eq!( + serde_json::to_value(&multiple).unwrap()["init_script"], + serde_json::json!(["echo one", "echo two"]) + ); + + let null: AppCompose = serde_json::from_value(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "init_script": null + })) + .unwrap(); + assert!(null.init_script.is_empty()); + assert!(serde_json::to_value(&null) + .unwrap() + .get("init_script") + .is_none()); + + assert!(serde_json::from_value::<AppCompose>(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "init_script": ["echo one", 2] + })) + .is_err()); + + assert!(serde_json::from_value::<AppCompose>(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "init_script": ["1", "2", "3", "4", "5", "6"] + })) + .unwrap_err() + .to_string() + .contains("at most 5")); + } + #[test] fn manifest_version_accepts_string_versions() { let compose = parse_compose(serde_json::json!("3")).unwrap(); @@ -401,6 +915,47 @@ mod app_compose_tests { assert_eq!(compose.manifest_version_u32(), Some(3)); } + #[test] + fn event_log_v1_is_omitted_but_v2_is_serialized() { + #[derive(Serialize)] + struct VersionField { + #[serde(skip_serializing_if = "EventLogVersion::is_v1")] + event_log_version: EventLogVersion, + } + + let v1 = serde_json::to_value(VersionField { + event_log_version: EventLogVersion::V1, + }) + .unwrap(); + assert!(v1.get("event_log_version").is_none()); + + let v2 = serde_json::to_value(VersionField { + event_log_version: EventLogVersion::V2, + }) + .unwrap(); + assert_eq!(v2["event_log_version"], 2); + } + + #[test] + fn parses_supported_container_snapshotters() { + let compose: AppCompose = serde_json::from_value(serde_json::json!({ + "manifest_version": "3", + "name": "test", + "runner": "nerdctl-compose", + "snapshotter": "stargz" + })) + .unwrap(); + assert_eq!(compose.snapshotter, Some(ContainerSnapshotter::Stargz)); + + let invalid = serde_json::from_value::<AppCompose>(serde_json::json!({ + "manifest_version": "3", + "name": "test", + "runner": "nerdctl-compose", + "snapshotter": "unknown" + })); + assert!(invalid.is_err()); + } + #[test] fn manifest_version_accepts_legacy_numeric_1_and_2() { assert_eq!( @@ -446,21 +1001,25 @@ mod app_compose_tests { } #[test] - fn requirements_support_os_version_and_platforms() { + fn requirements_support_platforms_and_policies() { let compose: AppCompose = serde_json::from_value(serde_json::json!({ "manifest_version": "3", "name": "test", "runner": "docker-compose", "requirements": { - "os_version": ">=0.6.1", "platforms": ["dstack-gcp-tdx", "dstack-tdx"], "tdx_measure_acpi_tables": true, - "launch_token_hash": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + "launch_token_hash": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + "gpu_policy": { + "rego": "package policy\n\ndefault nv_match = false\n", + "allow_devtools": true, + "allow_debug": true, + "allow_insecure_boot": true + } } })) .unwrap(); let requirements = compose.requirements.as_ref().unwrap(); - assert_eq!(requirements.os_version.as_deref(), Some(">=0.6.1")); assert_eq!( requirements.platforms, Some(vec!["dstack-gcp-tdx".to_string(), "dstack-tdx".to_string()]) @@ -470,13 +1029,22 @@ mod app_compose_tests { requirements.launch_token_hash.as_deref(), Some("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08") ); + let gpu_policy = &requirements.gpu_policy; + assert!(gpu_policy.attest_gpu); + assert_eq!( + gpu_policy.rego.as_deref(), + Some("package policy\n\ndefault nv_match = false\n") + ); + assert!(gpu_policy.allow_devtools); + assert!(gpu_policy.allow_debug); + assert!(gpu_policy.allow_insecure_boot); let err = serde_json::from_value::<AppCompose>(serde_json::json!({ "manifest_version": "3", "name": "test", "runner": "docker-compose", "requirements": { - "os_version_policy": ">=0.6.1" + "os_version": ">=0.6.1" } })) .unwrap_err(); @@ -494,7 +1062,10 @@ mod app_compose_tests { .unwrap(); let requirements = omitted.requirements.as_ref().unwrap(); assert_eq!(requirements.platforms, None); + assert!(requirements.gpu_policy.is_default()); assert!(requirements.is_empty()); + let serialized = serde_json::to_value(requirements).unwrap(); + assert!(serialized.get("gpu_policy").is_none()); let explicit_empty: AppCompose = serde_json::from_value(serde_json::json!({ "manifest_version": "3", @@ -534,54 +1105,96 @@ mod app_compose_tests { let requirements = launch_token.requirements.as_ref().unwrap(); assert!(requirements.launch_token_hash.is_some()); assert!(!requirements.is_empty()); + + let gpu_policy: AppCompose = serde_json::from_value(serde_json::json!({ + "manifest_version": "3", + "name": "test", + "runner": "docker-compose", + "requirements": { + "gpu_policy": { + "rego": "package policy\n\ndefault nv_match = false\n" + } + } + })) + .unwrap(); + let requirements = gpu_policy.requirements.as_ref().unwrap(); + let gpu_policy = &requirements.gpu_policy; + assert!(gpu_policy.attest_gpu); + assert!(gpu_policy.rego.is_some()); + assert!(!gpu_policy.allow_devtools); + assert!(!gpu_policy.allow_debug); + assert!(!gpu_policy.allow_insecure_boot); + assert!(!requirements.is_empty()); + + let err = serde_json::from_value::<AppCompose>(serde_json::json!({ + "manifest_version": "3", + "name": "test", + "runner": "docker-compose", + "requirements": { + "gpu_policy": { + "rego": "package policy", + "allow_debugger": true + } + } + })) + .unwrap_err(); + assert!(err.to_string().contains("unknown field")); } #[test] fn attest_gpu_defaults_to_true() { - let no_requirements: AppCompose = serde_json::from_value(serde_json::json!({ - "manifest_version": 2, + assert!(GpuPolicy::default().attest_gpu); + + let omitted: AppCompose = serde_json::from_value(serde_json::json!({ + "manifest_version": "3", "name": "test", - "runner": "docker-compose" + "runner": "docker-compose", + "requirements": {} })) .unwrap(); - assert!(no_requirements.attest_gpu()); + let requirements = omitted.requirements.as_ref().unwrap(); + assert!(requirements.gpu_policy.attest_gpu); + assert!(requirements.is_empty()); - let omitted: AppCompose = serde_json::from_value(serde_json::json!({ + let explicit_empty: AppCompose = serde_json::from_value(serde_json::json!({ "manifest_version": "3", "name": "test", "runner": "docker-compose", - "requirements": {} + "requirements": { + "gpu_policy": {} + } })) .unwrap(); - assert!(omitted.attest_gpu()); - assert!(omitted.requirements.as_ref().unwrap().is_empty()); + assert_eq!( + explicit_empty.requirements.unwrap().gpu_policy, + requirements.gpu_policy + ); let disabled: AppCompose = serde_json::from_value(serde_json::json!({ "manifest_version": "3", "name": "test", "runner": "docker-compose", "requirements": { - "attest_gpu": false + "gpu_policy": { + "attest_gpu": false + } } })) .unwrap(); - assert!(!disabled.attest_gpu()); let requirements = disabled.requirements.as_ref().unwrap(); - assert_eq!(requirements.attest_gpu, Some(false)); + assert!(!requirements.gpu_policy.attest_gpu); assert!(!requirements.is_empty()); - // The opt-out is ignored on legacy manifests (requirements are only - // valid on manifest_version >= 3; guests reject such composes anyway). - let legacy_optout: AppCompose = serde_json::from_value(serde_json::json!({ - "manifest_version": 2, + let old_location = serde_json::from_value::<AppCompose>(serde_json::json!({ + "manifest_version": "3", "name": "test", "runner": "docker-compose", "requirements": { "attest_gpu": false } })) - .unwrap(); - assert!(legacy_optout.attest_gpu()); + .unwrap_err(); + assert!(old_location.to_string().contains("unknown field")); } #[test] @@ -605,7 +1218,22 @@ pub struct SysConfig { pub kms_urls: Vec<String>, #[serde(default, alias = "tproxy_urls")] pub gateway_urls: Vec<String>, - pub pccs_url: Option<String>, + /// Independently operated gateway clusters. URLs within one entry are + /// failover endpoints for the same cluster. When empty, `gateway_urls` is + /// treated as one legacy cluster. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub gateway_clusters: Vec<GatewayClusterConfig>, + /// Backward-compatible input for sys-config files produced by older hosts. + #[serde(default, rename = "pccs_url", skip_serializing)] + legacy_pccs_url: Option<String>, + /// Attestation collateral service endpoints. Platform defaults are used + /// for fields that are absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub collateral_urls: Option<CollateralUrls>, + /// Optional NVIDIA attestation collateral proxy. When present, nvattest + /// fetches both OCSP responses and RIM documents through this endpoint. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub nvidia_attestation_proxy_url: Option<String>, pub docker_registry: Option<String>, pub host_api_url: Option<String>, /// MrConfigV3 document string for platform app/config binding. @@ -619,6 +1247,141 @@ pub struct SysConfig { pub vm_config: String, } +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct GatewayClusterConfig { + /// Stable local name used for the per-cluster key cache. + pub name: String, + /// Failover RPC endpoints belonging to this cluster. + pub urls: Vec<String>, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct CollateralUrls { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pccs: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub amd_kds: Option<String>, +} + +impl SysConfig { + pub fn collateral_urls(&self) -> CollateralUrls { + let mut urls = self.collateral_urls.clone().unwrap_or_default(); + if urls.pccs.is_none() { + urls.pccs.clone_from(&self.legacy_pccs_url); + } + urls + } +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +pub struct TeeSimulatorConfig { + /// Platform ABI exposed by dstack-tee-simulator. Defaults to `dstack-tdx`. + #[serde(default)] + pub platform: TeeVariant, + /// Hex-encoded 32-byte development PKI seed. The host collateral service + /// and guest simulator must receive the same seed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mock_attestation_seed: Option<String>, + /// Base URL used in mock collateral certificates (AIA/CRL). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub collateral_base_url: Option<String>, + /// MrConfigV3 document used to generate mock platform evidence. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mr_config: Option<String>, + /// JSON serialized VmConfig used to generate mock platform evidence. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vm_config: Option<String>, + /// Ordered SHA-384 PCR extensions used to reproduce the AWS boot state in + /// the development NitroTPM simulator. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub aws_pcr_replay: Option<AwsPcrReplay>, + /// Image-specific GCP TPM event log replayed by the development simulator. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gcp_tpm_replay: Option<GcpTpmReplay>, +} + +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)] +pub struct GcpTpmReplay { + #[serde(with = "serde_human_bytes::base64")] + pub event_log: Vec<u8>, +} + +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)] +pub struct AwsPcrReplay { + pub version: u32, + pub events: Vec<AwsPcrReplayEvent>, + #[serde(with = "hex_bytes")] + pub pcr4: Vec<u8>, + #[serde(with = "hex_bytes")] + pub pcr7: Vec<u8>, + #[serde(with = "hex_bytes")] + pub pcr12: Vec<u8>, +} + +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)] +pub struct AwsPcrReplayEvent { + pub pcr: u16, + pub event_type: String, + #[serde(with = "hex_bytes")] + pub digest: Vec<u8>, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Copy, Default, PartialEq, Eq, Encode, Decode)] +pub enum TeeVariant { + #[default] + #[serde(rename = "dstack-tdx")] + DstackTdx, + #[serde(rename = "dstack-gcp-tdx")] + DstackGcpTdx, + #[serde(rename = "dstack-nitro-enclave")] + DstackNitroEnclave, + #[serde(rename = "dstack-amd-sev-snp")] + DstackAmdSevSnp, + #[serde(rename = "dstack-aws-nitro-tpm")] + DstackAwsNitroTpm, +} + +impl TeeVariant { + pub fn has_tdx(self) -> bool { + matches!(self, Self::DstackTdx | Self::DstackGcpTdx) + } + + pub fn tpm_event_pcr_and_bank(self) -> Option<(u32, &'static str)> { + match self { + Self::DstackGcpTdx => Some((14, "sha256")), + Self::DstackAwsNitroTpm => Some((14, "sha384")), + Self::DstackTdx | Self::DstackAmdSevSnp | Self::DstackNitroEnclave => None, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::DstackTdx => "dstack-tdx", + Self::DstackGcpTdx => "dstack-gcp-tdx", + Self::DstackAmdSevSnp => "dstack-amd-sev-snp", + Self::DstackNitroEnclave => "dstack-nitro-enclave", + Self::DstackAwsNitroTpm => "dstack-aws-nitro-tpm", + } + } +} + +impl std::str::FromStr for TeeVariant { + type Err = String; + + fn from_str(value: &str) -> Result<Self, Self::Err> { + match value { + "dstack-tdx" => Ok(Self::DstackTdx), + "dstack-gcp-tdx" => Ok(Self::DstackGcpTdx), + "dstack-amd-sev-snp" => Ok(Self::DstackAmdSevSnp), + "dstack-nitro-enclave" => Ok(Self::DstackNitroEnclave), + "dstack-aws-nitro-tpm" => Ok(Self::DstackAwsNitroTpm), + _ => Err(format!("unsupported TEE variant: {value}")), + } + } +} + impl SysConfig { /// Canonical MrConfigV3 document for this VM, if any. /// @@ -652,6 +1415,14 @@ fn is_default_num_nics(n: &u32) -> bool { *n == default_num_nics() } +fn is_zero(n: &u32) -> bool { + *n == 0 +} + +fn is_false(value: &bool) -> bool { + !value +} + #[derive(Deserialize, Serialize, Debug, Clone)] pub struct VmConfig { #[serde(with = "hex_bytes", default)] @@ -685,6 +1456,15 @@ pub struct VmConfig { skip_serializing_if = "is_default_num_nics" )] pub num_nics: u32, + /// Number of read-only verity volume devices attached to the guest. Each + /// volume adds a virtio-blk PCI device before the NICs and therefore + /// changes the measured ACPI/DSDT layout. + #[serde(default, skip_serializing_if = "is_zero")] + pub num_verity_volumes: u32, + /// Whether QEMU attaches a software TPM device. The TPM changes the ACPI + /// table layout and must therefore be included in TDX measurement inputs. + #[serde(default, skip_serializing_if = "is_false")] + pub swtpm: bool, #[serde(default)] pub hotplug_off: bool, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -704,10 +1484,14 @@ pub struct VmConfig { #[serde(default, skip_serializing_if = "TdxAttestationVariant::is_legacy")] pub tdx_attestation_variant: TdxAttestationVariant, /// TDX-only no-image-download measurement material. Attached whenever - /// the OS image provides it, regardless of `tdx_attestation_variant`, so - /// a verifier can choose lite verification even for a boot that resolved - /// to `Legacy`. Omitted only when the image predates this measurement - /// material. + /// the OS image provides it, regardless of `tdx_attestation_variant`, and + /// omitted only when the image predates this measurement material. + /// + /// Its presence does not select lite verification: `tdx_attestation_variant` + /// alone does. A `Legacy` boot is verified through the image download even + /// when this document is attached, because the two paths disagree on what + /// `os_image_hash` means and honoring the document would move a boot the + /// app pinned to `Legacy` onto the weaker image-identity check. #[serde(default, skip_serializing_if = "Option::is_none")] pub tdx_measurement: Option<TdxOsImageMeasurementDocument>, /// GCP TDX no-image-download measurement material. Present for GCP @@ -1231,20 +2015,37 @@ impl SevOsImageMeasurementDocument { pub struct TdxOsImageMeasurement { pub image: TdxImageMeasurement, pub tdvf: TdxTdvfMeasurement, + /// Whether this image's OVMF normalizes the Linux setup header before + /// measuring the kernel, which decides what + /// [`TdxImageMeasurement::kernel_authenticode`] covers. + /// + /// Omitted from the CBOR when false, so a document from before the + /// normalization existed re-encodes byte for byte and keeps its + /// `os_image_hash`. + #[serde(default)] + pub kernel_header_normalized: bool, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct TdxImageMeasurement { - /// SHA-384 of the exact kernel command line event measured into RTMR[2]. + /// The image-provided kernel command line, without OVMF's `initrd=initrd` + /// suffix. /// - /// The measured value is the image-provided command line plus OVMF/QEMU's - /// `initrd=initrd` suffix, encoded as UTF-16LE with a trailing NUL. - #[serde(with = "hex_bytes")] - pub kernel_cmdline_sha384: Vec<u8>, - /// Authenticode SHA-384 digest of the QEMU-patched kernel image when the - /// guest memory is at or above QEMU's high-memory TDX initrd placement - /// threshold. Below that threshold the patched kernel header depends on the - /// exact guest memory size, so the no-image-download verifier rejects it. + /// The RTMR[2] command-line event is derived from this, so the document is + /// self-describing: a verifier that never downloads the image can still + /// report the effective command line and check what it pins, notably + /// `dstack.rootfs_hash`. This supersedes the `kernel_cmdline_sha384` digest + /// carried by version 3, which could not be turned back into a string. + pub base_cmdline: String, + /// Authenticode SHA-384 digest of the kernel image OVMF measures into + /// RTMR[1]. Which bytes that covers is + /// [`TdxOsImageMeasurement::kernel_header_normalized`]. + /// + /// When the header is not normalized it is QEMU's rewritten copy, computed + /// at or above QEMU's high-memory TDX initrd placement threshold; below + /// that threshold the rewritten header depends on the exact guest memory + /// size, so the no-image-download verifier rejects those sizes. When it is + /// normalized it is the kernel file as shipped, with no such restriction. #[serde(with = "hex_bytes")] pub kernel_authenticode: Vec<u8>, /// SHA-384 of the initrd file bytes. This is the second RTMR[2] event. @@ -1274,12 +2075,20 @@ pub struct TdxMrtdCandidates { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] struct CborTdxImageMeasurement { - /// Measured kernel cmdline SHA-384. - #[serde(rename = "cmdline_sha384", with = "hex_bytes")] - kernel_cmdline_sha384: Vec<u8>, - /// QEMU-patched kernel Authenticode SHA-384. + /// Image-provided kernel cmdline, without OVMF's `initrd=initrd` suffix. + /// Named to match `CborSevOsImageMeasurement`, which already carries the + /// equivalent string under the same key. + #[serde(rename = "cmdline")] + base_cmdline: String, + /// Kernel Authenticode SHA-384. Covers QEMU's rewritten copy, or the + /// kernel file as shipped when `kernel_header_normalized` is set. #[serde(with = "hex_bytes")] kernel_authenticode: Vec<u8>, + /// Whether the image's OVMF normalizes the Linux setup header before + /// measuring. Omitted when false, so documents from before this existed + /// encode and decode unchanged. + #[serde(default, skip_serializing_if = "is_false")] + kernel_header_normalized: bool, /// Initrd SHA-384. #[serde(with = "hex_bytes")] initrd_sha384: Vec<u8>, @@ -1314,8 +2123,9 @@ impl From<&TdxOsImageMeasurement> for CborTdxOsImageMeasurement { Self { version: TdxOsImageMeasurement::VERSION, image: CborTdxImageMeasurement { - kernel_cmdline_sha384: measurement.image.kernel_cmdline_sha384.clone(), + base_cmdline: measurement.image.base_cmdline.clone(), kernel_authenticode: measurement.image.kernel_authenticode.clone(), + kernel_header_normalized: measurement.kernel_header_normalized, initrd_sha384: measurement.image.initrd_sha384.clone(), }, tdvf: CborTdxTdvfMeasurement { @@ -1333,8 +2143,9 @@ impl From<&TdxOsImageMeasurement> for CborTdxOsImageMeasurement { impl From<CborTdxOsImageMeasurement> for TdxOsImageMeasurement { fn from(measurement: CborTdxOsImageMeasurement) -> Self { Self { + kernel_header_normalized: measurement.image.kernel_header_normalized, image: TdxImageMeasurement { - kernel_cmdline_sha384: measurement.image.kernel_cmdline_sha384, + base_cmdline: measurement.image.base_cmdline, kernel_authenticode: measurement.image.kernel_authenticode, initrd_sha384: measurement.image.initrd_sha384, }, @@ -1362,9 +2173,25 @@ pub struct TdxOsImageMeasurementDocument { } impl TdxOsImageMeasurement { - pub const VERSION: u32 = 3; + /// Version 4 replaced the `cmdline_sha384` digest with the `cmdline` + /// string. Version 3 only ever shipped in v0.6.0 release candidates, so it + /// is rejected rather than carried forward. + pub const VERSION: u32 = 4; + + /// `COMMAND_LINE_SIZE` on x86_64. A longer string cannot be the command + /// line any TDX guest booted with, so a document carrying one is malformed + /// rather than merely wrong. + /// + /// This is a contract check, not a resource guard: the document already + /// arrives inside an attestation bounded well below anything worth + /// defending against. What it buys is the error. Without it an oversized + /// string is measured like any other and rejected as an `RTMR2 mismatch`, + /// which points at the quote instead of at the document. It also restores + /// the length check the version 3 digest field carried. + const MAX_CMDLINE_LEN: usize = 2048; /// CBOR representation stored as `measurement.tdx.cbor`. + /// pub fn to_cbor_vec(&self) -> Vec<u8> { cbor_to_vec( &CborTdxOsImageMeasurement::from(self), @@ -1376,11 +2203,22 @@ impl TdxOsImageMeasurement { let cbor = cbor_from_slice::<CborTdxOsImageMeasurement>(bytes, "TdxOsImageMeasurement")?; if cbor.version != Self::VERSION { return Err(format!( - "TdxOsImageMeasurement: unsupported version {}, expected {}", + "TdxOsImageMeasurement: unsupported version {}, expected {}. \ + rebuild the image so measurement.tdx.cbor carries the kernel \ + command line, then re-register its os_image_hash.", cbor.version, Self::VERSION )); } + let cmdline_len = cbor.image.base_cmdline.len(); + if cmdline_len > Self::MAX_CMDLINE_LEN { + return Err(format!( + "TdxOsImageMeasurement: kernel command line is {} bytes, over the {} \ + byte x86_64 COMMAND_LINE_SIZE", + cmdline_len, + Self::MAX_CMDLINE_LEN + )); + } Ok(cbor.into()) } @@ -1578,6 +2416,11 @@ pub struct ImageInfo { /// fall back to version-based heuristics. #[serde(default, skip_serializing_if = "Option::is_none")] pub ovmf_variant: Option<OvmfVariant>, + /// Whether this image's OVMF normalizes the Linux setup header before + /// measuring the kernel, which decides what RTMR[1] covers. Absent on + /// every image built before that landed, and `false` is their behavior. + #[serde(default, skip_serializing_if = "is_false")] + pub kernel_header_normalized: bool, } pub mod mr_config; @@ -1620,6 +2463,7 @@ impl Platform { match product_name.map(str::trim) { Some("dstack" | "qemu") => return Some(Self::Dstack), Some("Google Compute Engine") => return Some(Self::Gcp), + Some("Nitro Enclave") => return Some(Self::NitroEnclave), _ => {} } @@ -1631,11 +2475,12 @@ impl Platform { /// Detect platform from system DMI information pub fn detect() -> Option<Self> { - // Nitro Enclave: NSM device exists only inside enclave + // `/dev/nsm` is the authoritative Nitro Enclave ABI. Check it before + // DMI because an enclave may inherit EC2-identifying DMI strings from + // its parent host, while a regular EC2 instance does not expose NSM. if Path::new("/dev/nsm").exists() { return Some(Self::NitroEnclave); } - let product_name = std::fs::read_to_string("/sys/class/dmi/id/product_name").ok(); let sys_vendor = std::fs::read_to_string("/sys/class/dmi/id/sys_vendor").ok(); Self::detect_from_dmi(product_name.as_deref(), sys_vendor.as_deref()) @@ -1676,10 +2521,18 @@ mod platform_tests { Some(Platform::Gcp) ); } + + #[test] + fn detects_nitro_enclave_from_simulated_dmi() { + assert_eq!( + Platform::detect_from_dmi(Some("Nitro Enclave"), Some("AWS Nitro Enclaves")), + Some(Platform::NitroEnclave) + ); + } } #[cfg(test)] -mod vm_config_num_nics_tests { +mod vm_config_device_count_tests { use super::VmConfig; fn legacy_json() -> serde_json::Value { @@ -1695,6 +2548,7 @@ mod vm_config_num_nics_tests { fn legacy_config_without_num_nics_defaults_to_one() { let cfg: VmConfig = serde_json::from_value(legacy_json()).unwrap(); assert_eq!(cfg.num_nics, 1); + assert_eq!(cfg.num_verity_volumes, 0); } #[test] @@ -1717,4 +2571,308 @@ mod vm_config_num_nics_tests { let serialized = serde_json::to_value(&cfg).unwrap(); assert_eq!(serialized.get("num_nics").and_then(|v| v.as_u64()), Some(2)); } + + #[test] + fn verity_volume_count_is_serialized_only_when_nonzero() { + let mut cfg: VmConfig = serde_json::from_value(legacy_json()).unwrap(); + let serialized = serde_json::to_value(&cfg).unwrap(); + assert!(serialized.get("num_verity_volumes").is_none()); + + cfg.num_verity_volumes = 2; + let serialized = serde_json::to_value(&cfg).unwrap(); + assert_eq!( + serialized + .get("num_verity_volumes") + .and_then(|v| v.as_u64()), + Some(2) + ); + } + + #[test] + fn swtpm_is_serialized_only_when_enabled() { + let mut cfg: VmConfig = serde_json::from_value(legacy_json()).unwrap(); + let serialized = serde_json::to_value(&cfg).unwrap(); + assert!(serialized.get("swtpm").is_none()); + + cfg.swtpm = true; + let serialized = serde_json::to_value(&cfg).unwrap(); + assert_eq!( + serialized.get("swtpm").and_then(|v| v.as_bool()), + Some(true) + ); + } +} + +#[cfg(test)] +mod log_sanitize_tests { + use super::sanitize_for_log; + + #[test] + fn strips_the_characters_that_forge_a_log_line() { + let forged = sanitize_for_log("web\nJan 01 INFO all good", 512); + assert!(!forged.contains('\n'), "{forged:?}"); + let escaped = sanitize_for_log("\u{1b}[2Jcleared", 512); + assert!(!escaped.contains('\u{1b}'), "{escaped:?}"); + } + + /// `char::is_control` is category Cc only, so these get through it -- and a + /// log viewer, or anything JSON downstream of one, treats them as breaks. + #[test] + fn strips_the_unicode_line_separators() { + for separator in ['\u{2028}', '\u{2029}', '\u{85}'] { + let out = sanitize_for_log(&format!("web{separator}forged"), 512); + assert!(!out.contains(separator), "{separator:?} survived: {out:?}"); + } + } + + /// U+202E reverses how everything after it renders, so an attacker decides + /// what a reader sees without controlling the bytes. + #[test] + fn strips_bidi_overrides() { + for bidi in ['\u{202E}', '\u{202A}', '\u{2066}', '\u{200F}', '\u{FEFF}'] { + let out = sanitize_for_log(&format!("web{bidi}txet"), 512); + assert!(!out.contains(bidi), "{bidi:?} survived: {out:?}"); + } + } + + /// Bounded in bytes, not characters. Counting characters lets a multi-byte + /// string reach four times the intended size. + #[test] + fn bounds_bytes_not_characters() { + let wide = "\u{1d54f}".repeat(400); + assert_eq!(wide.chars().count(), 400); + assert_eq!(wide.len(), 1600); + let out = sanitize_for_log(&wide, 512); + assert!(out.len() <= 512 + "... (truncated)".len(), "{}", out.len()); + } + + /// A bound that silently clips is worse than a short one: the reader cannot + /// tell a complete reason from a cut one. + #[test] + fn says_when_it_truncated() { + let wide = "\u{1d54f}".repeat(400); + assert!(sanitize_for_log(&wide, 512).ends_with("(truncated)")); + assert!(!sanitize_for_log("web is starting", 512).ends_with("(truncated)")); + } + + #[test] + fn never_splits_a_character() { + // 512 is not a multiple of 4, so a naive byte cut would split one. + let wide = "\u{1d54f}".repeat(400); + let out = sanitize_for_log(&wide, 512); + let body = out.trim_end_matches("... (truncated)"); + assert!(body.is_char_boundary(body.len())); + assert_eq!(body.len() % 4, 0); + } + + #[test] + fn ordinary_text_is_left_alone() { + assert_eq!(sanitize_for_log("web is starting", 512), "web is starting"); + } +} + +#[cfg(test)] +mod appcompose_sdk_parity { + use super::*; + + /// Every field `AppCompose` serializes, in sorted order. + /// + /// This exists to break when someone adds a field, so that the SDKs get + /// updated in the same change. What the SDKs compute is a compose hash that + /// gets whitelisted on chain, so a field they drop means the digest + /// describes an app-compose that is not the one being deployed -- and + /// nothing anywhere notices. + /// + /// All four are now *correct* without help: Go and Python keep unrecognised + /// keys in a catch-all, and JS's interface has an index signature and is + /// hashed from the object at runtime. So a missed field costs ergonomics + /// rather than correctness -- a user cannot name it with a type. That is + /// still worth fixing, in the same change, which is what this test is for. + /// + /// If it fails: add the field to `sdk/go/dstack/compose_hash.go`, + /// `sdk/js/src/get-compose-hash.ts` and + /// `sdk/python/src/dstack_sdk/get_compose_hash.py`, then update the list. + #[test] + fn every_serialized_field_is_accounted_for() { + let expected = [ + "allowed_envs", + "event_log_version", + "features", + "gateway_enabled", + "init_script", + "key_provider", + "key_provider_id", + "kms_enabled", + "local_key_provider_enabled", + "manifest_version", + "name", + "no_instance_id", + "port_policy", + "public_logs", + "public_sysinfo", + "public_tcbinfo", + "requirements", + "runner", + "secure_time", + "storage_discard", + "storage_fs", + "swap_size", + "verity_volumes", + ]; + + // Every optional field populated, so nothing is skipped by + // `skip_serializing_if`. + let populated: AppCompose = serde_json::from_value(serde_json::json!({ + "manifest_version": "3", + "name": "demo", + "runner": "docker-compose", + "docker_compose_file": "services: {}\n", + "init_script": ["a.sh"], + "storage_fs": "ext4", + "storage_discard": false, + "swap_size": "2G", + "event_log_version": 2, + "port_policy": {"ports": [{"port": 8080, "pp": true}], "restrict_mode": true}, + "verity_volumes": [{ + "source": "v.img", + "verity_root": "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff", + "target": "/mnt/v", + }], + "requirements": {"platforms": ["dstack-tdx"]}, + "snapshotter": "overlayfs", + })) + .expect("the fixture must stay parseable"); + + let serde_json::Value::Object(map) = serde_json::to_value(&populated).expect("serialize") + else { + panic!("AppCompose must serialize as an object"); + }; + let mut actual = map.keys().cloned().collect::<Vec<_>>(); + actual.sort(); + // Fields whose presence depends on the fixture rather than on the type. + actual.retain(|key| !matches!(key.as_str(), "docker_compose_file" | "snapshotter")); + + assert_eq!( + actual, expected, + "AppCompose gained or lost a field. The Go and JS SDKs declare this \ + type as a closed struct, so a field they do not know is silently \ + dropped from the compose hash they compute -- and that hash is what \ + gets whitelisted on chain. Update sdk/go and sdk/js, then this list." + ); + } +} + +#[cfg(test)] +mod image_info_tests { + use super::*; + + /// The image-download verification path reads this out of metadata.json to + /// decide which kernel bytes RTMR[1] covers. If the field goes missing here + /// the parse still succeeds and every normalized image is measured the old + /// way, which is a silent attestation failure. + #[test] + fn metadata_declares_whether_the_kernel_header_is_normalized() { + let base = r#"{"cmdline":"c","kernel":"bzImage","initrd":"i","bios":"b""#; + let normalized: ImageInfo = + serde_json::from_str(&format!("{base},\"kernel_header_normalized\":true}}")).unwrap(); + assert!(normalized.kernel_header_normalized); + + // Every image built before the field existed omits it. + let legacy: ImageInfo = serde_json::from_str(&format!("{base}}}")).unwrap(); + assert!(!legacy.kernel_header_normalized); + } +} + +#[cfg(test)] +mod tdx_measurement_cbor_tests { + use super::*; + + fn measurement(kernel_header_normalized: bool) -> TdxOsImageMeasurement { + TdxOsImageMeasurement { + kernel_header_normalized, + image: TdxImageMeasurement { + base_cmdline: "console=ttyS0 dstack.rootfs_hash=11".to_string(), + kernel_authenticode: vec![0x22; 48], + initrd_sha384: vec![0x33; 48], + }, + tdvf: TdxTdvfMeasurement { + ovmf_variant: OvmfVariant::default(), + mrtd: TdxMrtdCandidates { + single_pass: vec![0x44; 48], + two_pass: vec![0x55; 48], + }, + td_hob_witness: vec![0x01, 0x02, 0x03], + }, + } + } + + /// A command line longer than the kernel could ever have been handed is a + /// malformed document, and saying so beats measuring it and reporting an + /// RTMR[2] mismatch that points at the quote instead. + #[test] + fn an_oversized_command_line_is_rejected_by_name() { + let mut oversized = measurement(true); + oversized.image.base_cmdline = "a".repeat(TdxOsImageMeasurement::MAX_CMDLINE_LEN + 1); + let err = TdxOsImageMeasurement::from_cbor_slice(&oversized.to_cbor_vec()) + .expect_err("an oversized command line must not decode"); + assert!(err.contains("COMMAND_LINE_SIZE"), "{err}"); + + // The bound itself still decodes, so it rejects nothing a guest could + // actually have booted with. + let mut at_limit = measurement(true); + at_limit.image.base_cmdline = "a".repeat(TdxOsImageMeasurement::MAX_CMDLINE_LEN); + TdxOsImageMeasurement::from_cbor_slice(&at_limit.to_cbor_vec()) + .expect("the limit itself is valid"); + } + + /// Which kernel bytes the digest covers has to survive a round trip in + /// both directions. + #[test] + fn the_kernel_header_flag_round_trips() { + for normalized in [false, true] { + let original = measurement(normalized); + let decoded = TdxOsImageMeasurement::from_cbor_slice(&original.to_cbor_vec()).unwrap(); + assert_eq!(decoded, original); + assert_eq!(decoded.kernel_header_normalized, normalized); + } + } + + /// The flag is omitted when false, so a document from before the + /// normalization existed encodes to the same bytes it always did. Those + /// bytes are what `sha256sum.txt` -- and therefore `os_image_hash` -- + /// commits to, and the document shape did not change, so the version does + /// not move either. + #[test] + fn a_pre_normalization_document_does_not_drift() { + let cbor = measurement(false).to_cbor_vec(); + assert!( + !cbor.windows(24).any(|w| w == b"kernel_header_normalized"), + "the flag must not appear in a pre-normalization document" + ); + let value = TdxOsImageMeasurement::cbor_json_value_from_slice(&cbor).unwrap(); + assert_eq!( + value["version"], + serde_json::json!(TdxOsImageMeasurement::VERSION) + ); + let twice = TdxOsImageMeasurement::from_cbor_slice(&cbor) + .unwrap() + .to_cbor_vec(); + assert_eq!(cbor, twice); + } + + #[test] + fn unknown_versions_are_rejected() { + // CBOR stores the version as the single unsigned byte following the + // "version" key, so rewriting it forges another version. + let key = [0x67, b'v', b'e', b'r', b's', b'i', b'o', b'n']; + let mut cbor = measurement(true).to_cbor_vec(); + let at = cbor + .windows(key.len()) + .position(|window| window == key) + .expect("encoded document contains a version key"); + cbor[at + key.len()] = 2; + + let err = TdxOsImageMeasurement::from_cbor_slice(&cbor).unwrap_err(); + assert!(err.contains("unsupported version 2"), "unexpected: {err}"); + } } diff --git a/dstack/dstack-types/src/mr_config.rs b/dstack/dstack-types/src/mr_config.rs index f5f8d09de..ce38f4e7c 100644 --- a/dstack/dstack-types/src/mr_config.rs +++ b/dstack/dstack-types/src/mr_config.rs @@ -5,6 +5,7 @@ use or_panic::ResultOrPanic; use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; +use serde_with::skip_serializing_none; use sha2::Sha256; use sha3::{Digest, Keccak256}; use std::{error::Error, fmt}; @@ -100,40 +101,58 @@ impl From<serde_json::Error> for MrConfigDocumentError { } } +#[skip_serializing_none] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct MrConfigV3 { #[serde(default = "mr_config_v3_version")] pub version: u8, - #[serde(with = "hex_bytes")] - pub app_id: Vec<u8>, + /// Optional application identity pin. + #[serde(default, with = "hex_bytes")] + pub app_id: Option<Vec<u8>>, #[serde(with = "hex_bytes")] pub compose_hash: Vec<u8>, + /// Hash of the raw application GPU policy. GPU launches populate it; + /// non-GPU and historical v3 launch documents omit it. + #[serde(default, with = "hex_bytes")] + pub gpu_policy_hash: Option<Vec<u8>>, pub key_provider: KeyProviderKind, #[serde(default, with = "hex_bytes")] - pub key_provider_id: Vec<u8>, + pub key_provider_id: Option<Vec<u8>>, #[serde(default, with = "hex_bytes")] - pub instance_id: Vec<u8>, + pub instance_id: Option<Vec<u8>>, + /// Optional SHA-256 pins for init scripts, in execution order. An omitted + /// field disables this check; an empty list requires no init scripts. + #[serde(default, with = "crate::init_script_hashes::option")] + pub init_script_hashes: Option<Vec<Vec<u8>>>, } impl MrConfigV3 { pub fn new( app_id: Vec<u8>, compose_hash: Vec<u8>, + gpu_policy_hash: Option<Vec<u8>>, key_provider: KeyProviderKind, key_provider_id: Vec<u8>, instance_id: Vec<u8>, ) -> Self { Self { version: mr_config_v3_version(), - app_id, + app_id: (!app_id.is_empty()).then_some(app_id), compose_hash, + gpu_policy_hash, key_provider, - key_provider_id, - instance_id, + key_provider_id: (!key_provider_id.is_empty()).then_some(key_provider_id), + instance_id: (!instance_id.is_empty()).then_some(instance_id), + init_script_hashes: None, } } + pub fn with_init_script_hashes(mut self, init_script_hashes: Vec<Vec<u8>>) -> Self { + self.init_script_hashes = Some(init_script_hashes); + self + } + pub fn to_snp_host_data(&self) -> [u8; 32] { Self::snp_host_data_from_document(&self.to_canonical_json()) } @@ -191,17 +210,76 @@ mod tests { let config = MrConfigV3::new( vec![0x11; 20], vec![0x22; 32], + Some(vec![0x55; 32]), KeyProviderKind::Kms, vec![0x33; 32], vec![0x44; 20], ); let mut changed = config.clone(); - changed.app_id[0] ^= 0xff; + changed.app_id.as_mut().expect("app_id is set")[0] ^= 0xff; assert_ne!(config.to_snp_host_data(), changed.to_snp_host_data()); assert_eq!(config.to_snp_host_data().len(), 32); assert_ne!(config.to_tdx_mr_config_id(), changed.to_tdx_mr_config_id()); assert_eq!(config.to_tdx_mr_config_id()[0], 3); + + let mut changed = config.clone(); + if let Some(gpu_policy_hash) = &mut changed.gpu_policy_hash { + gpu_policy_hash[0] ^= 0xff; + } + assert_ne!(config.to_snp_host_data(), changed.to_snp_host_data()); + assert_ne!(config.to_tdx_mr_config_id(), changed.to_tdx_mr_config_id()); + } + + #[test] + fn mr_config_v3_omits_gpu_policy_hash_for_non_gpu_launches() -> Result<(), Box<dyn Error>> { + let config = MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + None, + KeyProviderKind::None, + Vec::new(), + vec![0x44; 20], + ); + let document = config.to_canonical_json(); + + assert!(!document.contains("gpu_policy_hash")); + assert_eq!(MrConfigV3::from_document(&document)?, config); + Ok(()) + } + + #[test] + fn mr_config_v3_defaults_missing_app_id_to_empty() -> Result<(), Box<dyn Error>> { + let config = MrConfigV3::from_document( + r#"{"compose_hash":"2222222222222222222222222222222222222222222222222222222222222222","key_provider":"none"}"#, + )?; + + assert!(config.app_id.is_none()); + assert!(!config.to_canonical_json().contains("app_id")); + Ok(()) + } + + #[test] + fn mr_config_v3_binds_ordered_init_script_hashes() -> Result<(), Box<dyn Error>> { + let config = MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + None, + KeyProviderKind::None, + Vec::new(), + vec![0x44; 20], + ) + .with_init_script_hashes(vec![vec![0xaa; 32], vec![0xbb; 32]]); + let document = config.to_canonical_json(); + let decoded = MrConfigV3::from_document(&document)?; + + assert_eq!(decoded.init_script_hashes, config.init_script_hashes); + let reordered = MrConfigV3 { + init_script_hashes: Some(vec![vec![0xbb; 32], vec![0xaa; 32]]), + ..config.clone() + }; + assert_ne!(config.to_snp_host_data(), reordered.to_snp_host_data()); + Ok(()) } #[test] @@ -209,6 +287,7 @@ mod tests { let config = MrConfigV3::new( vec![0x11; 20], vec![0x22; 32], + Some(vec![0x55; 32]), KeyProviderKind::Kms, vec![0x33; 32], vec![0x44; 20], @@ -220,6 +299,7 @@ mod tests { concat!( "{\"app_id\":\"1111111111111111111111111111111111111111\",", "\"compose_hash\":\"2222222222222222222222222222222222222222222222222222222222222222\",", + "\"gpu_policy_hash\":\"5555555555555555555555555555555555555555555555555555555555555555\",", "\"instance_id\":\"4444444444444444444444444444444444444444\",", "\"key_provider\":\"kms\",", "\"key_provider_id\":\"3333333333333333333333333333333333333333333333333333333333333333\",", diff --git a/dstack/dstack-types/src/shared_filenames.rs b/dstack/dstack-types/src/shared_filenames.rs index a46afbdf9..79944cff6 100644 --- a/dstack/dstack-types/src/shared_filenames.rs +++ b/dstack/dstack-types/src/shared_filenames.rs @@ -5,6 +5,7 @@ pub const APP_COMPOSE: &str = "app-compose.json"; pub const APP_KEYS: &str = ".appkeys.json"; pub const SYS_CONFIG: &str = ".sys-config.json"; +pub const TEE_SIMULATOR_CONFIG: &str = ".tee-simulator.json"; pub const USER_CONFIG: &str = ".user-config"; pub const ENCRYPTED_ENV: &str = ".encrypted-env"; pub const DECRYPTED_ENV: &str = ".decrypted-env"; diff --git a/dstack/dstack-types/src/version.rs b/dstack/dstack-types/src/version.rs index 1ed3f226e..5e59704ea 100644 --- a/dstack/dstack-types/src/version.rs +++ b/dstack/dstack-types/src/version.rs @@ -52,8 +52,13 @@ impl Version { let mut parts = version.split('.'); let major = parts.next()?.parse().ok()?; let minor = parts.next()?.parse().ok()?; - // Patch is optional, defaults to 0 - let patch = parts.next().and_then(|s| s.parse().ok()).unwrap_or(0); + // Patch is optional and defaults to 0, but a patch that is *present* + // must be numeric: silently reading "0.6.garbage" as 0.6.0 would + // downgrade a malformed version into a valid-looking one. + let patch = match parts.next() { + Some(patch) => patch.parse().ok()?, + None => 0, + }; // Ignore extra parts like "0.5.6.1" Some(Self { @@ -160,6 +165,33 @@ mod tests { assert!(Version::parse("1").is_none()); assert!(Version::parse("v").is_none()); assert!(Version::parse("abc.def.ghi").is_none()); + // A leading `v` is not stripped: callers that accept tags must do it. + assert!(Version::parse("v0.5.6").is_none()); + // An image-name fragment must not parse as a version. + assert!(Version::parse("nvidia-0.6.0").is_none()); + } + + #[test] + fn test_parse_rejects_present_but_non_numeric_patch() { + // An absent patch defaults to 0 ... + assert_eq!(Version::parse("1.2").unwrap(), Version::new(1, 2, 0)); + // ... but a present one must be numeric rather than coerced to 0. + assert!(Version::parse("1.2.invalid").is_none()); + assert!(Version::parse("0.6.garbage.1").is_none()); + } + + #[test] + fn test_prerelease_compares_equal_to_release() { + // A release candidate is built from the code of the version it is a + // candidate for, so feature gates must treat the two as equal. + assert_eq!(Version::parse("0.6.1-rc1"), Version::parse("0.6.1")); + assert_eq!(Version::parse("0.6.1.rc1"), Version::parse("0.6.1")); + assert_eq!(Version::parse("0.6.1+build.5"), Version::parse("0.6.1")); + assert_eq!( + Version::parse("0.6.1-rc.1+build.5"), + Version::parse("0.6.1") + ); + assert!(Version::parse("0.6.0-rc1").unwrap() >= Version::new(0, 6, 0)); } #[test] diff --git a/dstack/dstack-types/tests/sdk_canonical_json.rs b/dstack/dstack-types/tests/sdk_canonical_json.rs new file mode 100644 index 000000000..bc12b7b92 --- /dev/null +++ b/dstack/dstack-types/tests/sdk_canonical_json.rs @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +#[test] +fn canonical_sdk_json_round_trips() { + // Byte-for-byte what the Python/Go/JS SDKs emit for this manifest; all + // three hash it to 81f9e7d7... The guest hashes the file it is given, so + // agreement means the guest must accept exactly these bytes and re-emit + // them unchanged. + let canonical = r#"{"docker_compose_file":"services:\n w:\n image: x && y\n","kms_enabled":true,"manifest_version":"3","name":"a","requirements":{"health_check":true,"health_status_file":"/dstack/health","launch_token_hash":"ff00"},"runner":"docker-compose"}"#; + let compose: dstack_types::AppCompose = + serde_json::from_str(canonical).expect("guest parses SDK output"); + let requirements = compose.requirements.as_ref().expect("requirements"); + assert!(requirements.health_check); + assert_eq!( + requirements.health_status_file.as_deref(), + Some("/dstack/health") + ); +} + +#[test] +fn the_nested_shape_is_rejected() { + // The shape this replaced. Rejected rather than ignored, so an app-compose + // written against a pre-release SDK cannot deploy and silently lose its + // health gating. + let nested = r#"{"runner":"bash","requirements":{"health_check":{"enabled":true}}}"#; + assert!(serde_json::from_str::<dstack_types::AppCompose>(nested).is_err()); +} diff --git a/dstack/dstack-util/Cargo.toml b/dstack/dstack-util/Cargo.toml index e0f04bf75..2fdb9ae52 100644 --- a/dstack/dstack-util/Cargo.toml +++ b/dstack/dstack-util/Cargo.toml @@ -10,6 +10,7 @@ edition.workspace = true license.workspace = true [dependencies] +nvattest.workspace = true aes-gcm.workspace = true anyhow.workspace = true base64.workspace = true @@ -24,12 +25,13 @@ scale = { workspace = true, features = ["derive"] } schnorrkel.workspace = true serde.workspace = true serde-human-bytes.workspace = true -semver.workspace = true serde_json.workspace = true +sd-notify.workspace = true sha2.workspace = true tokio = { workspace = true, features = ["full"] } tracing.workspace = true tracing-subscriber.workspace = true +url.workspace = true x25519-dalek.workspace = true dstack-kms-rpc.workspace = true @@ -41,21 +43,26 @@ tpm-attest.workspace = true tpm2.workspace = true tpm-qvl = { workspace = true, features = ["crl-download"] } host-api = { workspace = true, features = ["client"] } +guest-api.workspace = true cmd_lib.workspace = true toml.workspace = true dcap-qvl.workspace = true k256 = { workspace = true, features = ["ecdsa"] } dstack-types.workspace = true rand.workspace = true +regorus.workspace = true sha3.workspace = true dstack-attest.workspace = true cert-client.workspace = true x509-parser.workspace = true yaml-rust2.workspace = true bollard.workspace = true +binrw.workspace = true sodiumbox.workspace = true libc.workspace = true luks2.workspace = true +nvml-wrapper.workspace = true +lspci.workspace = true scopeguard.workspace = true tempfile.workspace = true ez-hash.workspace = true @@ -64,4 +71,5 @@ safe-write.workspace = true errify.workspace = true [dev-dependencies] +base64.workspace = true rand.workspace = true diff --git a/dstack/dstack-util/src/crypto.rs b/dstack/dstack-util/src/crypto.rs index c56a1682a..177de6adc 100644 --- a/dstack/dstack-util/src/crypto.rs +++ b/dstack/dstack-util/src/crypto.rs @@ -3,12 +3,36 @@ // SPDX-License-Identifier: Apache-2.0 use aes_gcm::{ - aead::{Aead, Nonce}, + aead::{Aead, Nonce, Payload}, Aes256Gcm, KeyInit, }; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, ensure, Context, Result}; +use binrw::{binrw, io::NoSeek, BinRead, BinWrite}; +use std::io::{Cursor, Read, Write}; use x25519_dalek::{PublicKey, StaticSecret}; +pub const STREAM_MAGIC: &[u8; 8] = b"dstkscrt"; +pub const DEFAULT_CHUNK_SIZE: usize = 1024 * 1024; +pub const MAX_CHUNK_SIZE: usize = 16 * 1024 * 1024; +const STREAM_VERSION: u8 = 0; +const FINAL_CHUNK: u8 = 1; + +#[binrw] +#[brw(little)] +struct StreamHeader { + version: u8, + ephemeral_public_key: [u8; 32], + nonce_prefix: [u8; 8], + chunk_size: u32, +} + +#[binrw] +#[brw(little)] +struct FrameHeader { + flags: u8, + plaintext_len: u32, +} + pub fn dh_agree(secret: [u8; 32], their_pubkey: [u8; 32]) -> [u8; 32] { let secret = StaticSecret::from(secret); let their_public = PublicKey::from(their_pubkey); @@ -20,13 +44,13 @@ pub fn dh_decrypt(secret: [u8; 32], ciphertext: &[u8]) -> Result<Vec<u8>> { // Extract components (matching JS implementation) let ephemeral_pubkey = ciphertext .get(..32) - .ok_or(anyhow!("Invalid ephemeral public key length"))? + .ok_or(anyhow!("invalid ephemeral public key length"))? .try_into() - .map_err(|_| anyhow!("Invalid ephemeral public key length"))?; - let iv = &ciphertext.get(32..44).ok_or(anyhow!("Invalid IV length"))?; + .map_err(|_| anyhow!("invalid ephemeral public key length"))?; + let iv = &ciphertext.get(32..44).ok_or(anyhow!("invalid IV length"))?; let ciphertext = &ciphertext .get(44..) - .ok_or(anyhow!("Invalid ciphertext length"))?; + .ok_or(anyhow!("invalid ciphertext length"))?; // Derive shared secret using X25519 let shared_secret = dh_agree(secret, ephemeral_pubkey); @@ -36,7 +60,7 @@ pub fn dh_decrypt(secret: [u8; 32], ciphertext: &[u8]) -> Result<Vec<u8>> { // Create AES-GCM cipher let cipher = Aes256Gcm::new_from_slice(&shared_secret) - .map_err(|e| anyhow!("Failed to create cipher: {}", e))?; + .map_err(|e| anyhow!("failed to create cipher: {}", e))?; // Decrypt using AES-GCM cipher @@ -83,6 +107,200 @@ pub fn dh_encrypt(their_pubkey: [u8; 32], plaintext: &[u8]) -> Result<Vec<u8>> { Ok(out) } +fn stream_nonce(prefix: &[u8; 8], index: u32) -> [u8; 12] { + let mut nonce = [0u8; 12]; + nonce[..8].copy_from_slice(prefix); + nonce[8..].copy_from_slice(&index.to_be_bytes()); + nonce +} + +fn stream_aad(header: &StreamHeader, index: u32, frame_header: &FrameHeader) -> Result<Vec<u8>> { + let mut aad = Cursor::new(Vec::new()); + aad.write_all(STREAM_MAGIC) + .context("failed to encode stream magic as AAD")?; + header + .write(&mut aad) + .context("failed to encode stream header as AAD")?; + index + .write_le(&mut aad) + .context("failed to encode chunk index as AAD")?; + frame_header + .write(&mut aad) + .context("failed to encode frame header as AAD")?; + Ok(aad.into_inner()) +} + +/// Encrypts a reader as independently authenticated chunks. +pub fn dh_encrypt_stream( + remote_public_key: [u8; 32], + mut input: impl Read, + mut output: impl Write, + chunk_size: usize, +) -> Result<()> { + ensure!( + (1..=MAX_CHUNK_SIZE).contains(&chunk_size), + "chunk size must be between 1 and {MAX_CHUNK_SIZE} bytes" + ); + + let mut ephemeral_secret = [0u8; 32]; + getrandom::fill(&mut ephemeral_secret).context("failed to generate ephemeral secret")?; + let ephemeral_secret = StaticSecret::from(ephemeral_secret); + let ephemeral_public_key = PublicKey::from(&ephemeral_secret).to_bytes(); + let remote_public_key = PublicKey::from(remote_public_key); + let shared_secret = ephemeral_secret + .diffie_hellman(&remote_public_key) + .to_bytes(); + ensure!( + !shared_secret.iter().all(|byte| *byte == 0), + "invalid X25519 shared secret" + ); + let cipher = Aes256Gcm::new_from_slice(&shared_secret) + .map_err(|e| anyhow!("failed to create cipher: {e}"))?; + + let mut nonce_prefix = [0u8; 8]; + getrandom::fill(&mut nonce_prefix).context("failed to generate nonce prefix")?; + let header = StreamHeader { + version: STREAM_VERSION, + ephemeral_public_key, + nonce_prefix, + chunk_size: chunk_size as u32, + }; + output + .write_all(STREAM_MAGIC) + .context("failed to write stream magic")?; + header + .write(&mut NoSeek::new(&mut output)) + .context("failed to write stream header")?; + + let mut current = vec![0u8; chunk_size]; + let mut next = vec![0u8; chunk_size]; + let mut current_len = read_chunk(&mut input, &mut current)?; + let mut index = 0u32; + loop { + let next_len = read_chunk(&mut input, &mut next)?; + let final_chunk = next_len == 0; + let flags = if final_chunk { FINAL_CHUNK } else { 0 }; + let frame_header = FrameHeader { + flags, + plaintext_len: current_len as u32, + }; + let aad = stream_aad(&header, index, &frame_header)?; + let nonce = stream_nonce(&nonce_prefix, index); + let encrypted = cipher + .encrypt( + (&nonce).into(), + Payload { + msg: ¤t[..current_len], + aad: &aad, + }, + ) + .map_err(|e| anyhow!("failed to encrypt chunk {index}: {e}"))?; + frame_header + .write(&mut NoSeek::new(&mut output)) + .with_context(|| format!("failed to write header for chunk {index}"))?; + output + .write_all(&encrypted) + .with_context(|| format!("failed to write chunk {index}"))?; + if final_chunk { + break; + } + index = index.checked_add(1).context("too many chunks")?; + std::mem::swap(&mut current, &mut next); + current_len = next_len; + } + output.flush().context("failed to flush encrypted output")?; + Ok(()) +} + +fn read_chunk(input: &mut impl Read, buffer: &mut [u8]) -> Result<usize> { + let mut read = 0; + while read < buffer.len() { + match input + .read(&mut buffer[read..]) + .context("failed to read input")? + { + 0 => break, + n => read += n, + } + } + Ok(read) +} + +/// Decrypts a chunked stream after the caller has consumed [`STREAM_MAGIC`]. +pub fn dh_decrypt_stream( + secret: [u8; 32], + mut input: impl Read, + mut output: impl Write, +) -> Result<()> { + let header = + StreamHeader::read(&mut NoSeek::new(&mut input)).context("invalid stream header")?; + ensure!( + header.version == STREAM_VERSION, + "unsupported stream version: {}", + header.version + ); + let chunk_size = header.chunk_size as usize; + ensure!( + (1..=MAX_CHUNK_SIZE).contains(&chunk_size), + "invalid chunk size: {chunk_size}" + ); + + let shared_secret = dh_agree(secret, header.ephemeral_public_key); + ensure!( + !shared_secret.iter().all(|byte| *byte == 0), + "invalid X25519 shared secret" + ); + let cipher = Aes256Gcm::new_from_slice(&shared_secret) + .map_err(|e| anyhow!("failed to create cipher: {e}"))?; + + let mut index = 0u32; + loop { + let frame_header = FrameHeader::read(&mut NoSeek::new(&mut input)) + .with_context(|| format!("missing final chunk at chunk {index}"))?; + ensure!( + frame_header.flags & !FINAL_CHUNK == 0, + "invalid chunk flags" + ); + let final_chunk = frame_header.flags == FINAL_CHUNK; + let plaintext_len = frame_header.plaintext_len as usize; + ensure!(plaintext_len <= chunk_size, "chunk {index} is too large"); + ensure!( + final_chunk || plaintext_len == chunk_size, + "non-final chunk {index} has an invalid length" + ); + + let mut encrypted = vec![0u8; plaintext_len + 16]; + input + .read_exact(&mut encrypted) + .with_context(|| format!("truncated chunk {index}"))?; + let nonce = stream_nonce(&header.nonce_prefix, index); + let aad = stream_aad(&header, index, &frame_header)?; + let plaintext = cipher + .decrypt( + (&nonce).into(), + Payload { + msg: &encrypted, + aad: &aad, + }, + ) + .map_err(|e| anyhow!("failed to decrypt chunk {index}: {e}"))?; + output + .write_all(&plaintext) + .with_context(|| format!("failed to write chunk {index}"))?; + + if final_chunk { + let mut trailing = [0u8; 1]; + ensure!( + input.read(&mut trailing).context("failed to read input")? == 0, + "trailing data after final chunk" + ); + output.flush().context("failed to flush plaintext output")?; + return Ok(()); + } + index = index.checked_add(1).context("too many chunks")?; + } +} + #[cfg(test)] mod tests { use super::*; @@ -137,4 +355,57 @@ mod tests { let decrypted_str = String::from_utf8(decrypted).unwrap(); assert_eq!(decrypted_str, "[{\"key\":\"\",\"value\":\"\"}]"); } + + #[test] + fn test_stream_roundtrip() { + let secret = StaticSecret::random_from_rng(rand::thread_rng()); + let public_key = PublicKey::from(&secret).to_bytes(); + let plaintext = vec![0x5a; 2500]; + let mut encrypted = Vec::new(); + dh_encrypt_stream(public_key, plaintext.as_slice(), &mut encrypted, 1024).unwrap(); + assert_eq!(&encrypted[..STREAM_MAGIC.len()], STREAM_MAGIC); + + let mut decrypted = Vec::new(); + dh_decrypt_stream( + secret.to_bytes(), + &encrypted[STREAM_MAGIC.len()..], + &mut decrypted, + ) + .unwrap(); + assert_eq!(decrypted, plaintext); + } + + #[test] + fn test_stream_rejects_tampering_and_truncation() { + let secret = StaticSecret::random_from_rng(rand::thread_rng()); + let public_key = PublicKey::from(&secret).to_bytes(); + let mut encrypted = Vec::new(); + dh_encrypt_stream(public_key, b"hello".as_slice(), &mut encrypted, 4).unwrap(); + + let mut unknown_version = encrypted.clone(); + unknown_version[STREAM_MAGIC.len()] = STREAM_VERSION + 1; + assert!(dh_decrypt_stream( + secret.to_bytes(), + &unknown_version[STREAM_MAGIC.len()..], + Vec::new(), + ) + .is_err()); + + let mut tampered = encrypted.clone(); + *tampered.last_mut().unwrap() ^= 1; + assert!(dh_decrypt_stream( + secret.to_bytes(), + &tampered[STREAM_MAGIC.len()..], + Vec::new(), + ) + .is_err()); + + encrypted.truncate(encrypted.len() - 1); + assert!(dh_decrypt_stream( + secret.to_bytes(), + &encrypted[STREAM_MAGIC.len()..], + Vec::new(), + ) + .is_err()); + } } diff --git a/dstack/dstack-util/src/gateway_checker.rs b/dstack/dstack-util/src/gateway_checker.rs new file mode 100644 index 000000000..6bb20276f --- /dev/null +++ b/dstack/dstack-util/src/gateway_checker.rs @@ -0,0 +1,693 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Keeps this CVM's dstack-gateway registration alive after boot. +//! +//! Boot registers the CVM once, but that registration is not durable: the +//! gateway can be restarted, the WireGuard peers can change, and — since boot +//! no longer treats a gateway outage as fatal — the CVM may reach the +//! application start with no route at all. This supervisor closes that gap by +//! re-running the same registration whenever the observable state says it is +//! needed. +//! +//! There are exactly three reasons to refresh. The order matters, because +//! [`HANDSHAKE_TIMEOUT`] equals [`REFRESH_INTERVAL`] and the deadlines collide: +//! +//! 1. **No WireGuard config.** Boot-time registration never succeeded, so the +//! CVM has no route. Retried on a backoff (see [`Backoff`]). +//! 2. **Stale WireGuard handshake.** The tunnel exists but the peer stopped +//! answering for [`HANDSHAKE_TIMEOUT`], which usually means the gateway +//! restarted and forgot us. Checked *before* the periodic refresh and rate +//! limited to one attempt per timeout, because only this forced path can +//! rebuild a tunnel whose config is unchanged, and it is the expensive one. +//! 3. **Periodic re-registration.** The gateway expires idle registrations, so +//! re-register every [`REFRESH_INTERVAL`] even when everything looks fine. +//! +//! The decision logic is a pure function of an [`Observation`] so it can be +//! unit tested without a gateway, a KMS, or a WireGuard interface; all I/O +//! lives in [`cmd_gateway_checker`]. + +use std::path::PathBuf; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use cmd_lib::run_fun as cmd; +use sd_notify::NotifyState; +use tracing::{error, info, warn}; + +use crate::system_setup::GatewayRefresher; + +/// How often the loop samples the world. +const POLL_INTERVAL: Duration = Duration::from_secs(10); +/// Unconditional re-registration interval. +const REFRESH_INTERVAL: i64 = 180; +/// A tunnel with no handshake for this long is considered dead. +const HANDSHAKE_TIMEOUT: i64 = 180; +/// First retry delay when the CVM has no WireGuard config at all. +const MISSING_CONFIG_RETRY_INTERVAL: i64 = 30; +/// Upper bound for the retry backoff. +const MAX_RETRY_INTERVAL: i64 = 120; + +/// Exit code meaning "the gateway config is broken in a way retrying cannot +/// fix". Pinned by `RestartPreventExitStatus` in dstack-gateway-checker.service, so +/// changing it requires changing the unit too. +const EXIT_MISCONFIGURED: i32 = 3; + +#[derive(clap::Parser)] +/// Keep the dstack-gateway registration fresh +pub struct GatewayCheckerArgs { + /// dstack work directory + #[arg(long)] + work_dir: PathBuf, +} + +/// Exponential backoff over consecutive refresh failures. +/// +/// A refresh is not cheap: on a cold cache it costs a KMS round-trip, two +/// certificate signing requests and a TDX quote. A gateway outage is typically +/// fleet-wide, so a fixed short retry interval would have every CVM hammering +/// the KMS in lockstep and turn a gateway outage into a KMS outage. Backing off +/// to [`MAX_RETRY_INTERVAL`] keeps recovery prompt while bounding that load. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +struct Backoff { + consecutive_failures: u32, +} + +impl Backoff { + /// Minimum seconds that must elapse after the last attempt before the next + /// one. Zero while the last attempt succeeded. + fn delay(&self) -> i64 { + match self.consecutive_failures { + 0 => 0, + n => { + let shift = (n - 1).min(u32::BITS - 1); + MISSING_CONFIG_RETRY_INTERVAL + .checked_shl(shift) + .unwrap_or(MAX_RETRY_INTERVAL) + .min(MAX_RETRY_INTERVAL) + } + } + } + + fn record(&mut self, succeeded: bool) { + self.consecutive_failures = if succeeded { + 0 + } else { + self.consecutive_failures.saturating_add(1) + }; + } +} + +/// Everything the decision logic is allowed to look at. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Observation { + /// Seconds since the UNIX epoch. + now: i64, + /// Whether at least one `/etc/wireguard/dstack-wg*.conf` exists. + config_present: bool, + /// Most recent handshake as a UNIX timestamp; `None` if the interface has + /// never completed one (or does not exist yet). + latest_handshake: Option<i64>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Refresh { + /// Force `wg-quick` to be reapplied even if the rendered config is + /// byte-identical to what is already on disk. + force: bool, + reason: &'static str, +} + +#[derive(Debug, Default)] +struct Checker { + /// Timestamp of the last refresh attempt; `None` if none has run yet. + last_attempt: Option<i64>, + /// Timestamp of the last *forced* refresh; `None` if none has run yet. + /// Tracked separately from `last_attempt` so a cheap periodic refresh does + /// not re-arm the rate limit that protects the expensive forced path. + last_force: Option<i64>, + /// When the tunnel was first seen without any handshake; `None` while a + /// handshake exists or the config is absent. Means exactly one thing: when + /// we first observed a peer with no handshake. + handshake_missing_since: Option<i64>, + backoff: Backoff, +} + +const MISSING_CONFIG: Refresh = Refresh { + force: true, + reason: "WireGuard config is missing", +}; +const STALE_HANDSHAKE: Refresh = Refresh { + force: true, + reason: "WireGuard handshake is stale", +}; +const PERIODIC: Refresh = Refresh { + force: false, + reason: "periodic re-registration", +}; + +impl Checker { + /// Build the starting state for a checker coming up at `now`. + /// + /// If the WireGuard config is already on disk, boot registered this CVM + /// moments ago: `/etc` is a volatile overlay, so the file can only exist + /// because *this* boot wrote it. Start the periodic clock from now instead + /// of re-registering immediately. Otherwise a fleet rebooting together + /// would hit the gateway with a second full round of registrations seconds + /// after the first — piling onto the component this checker exists to + /// tolerate the loss of. With no config, boot's registration failed and + /// recovering fast is the whole point, so leave the clock unset and act on + /// the first poll. + fn starting(now: i64, config_present: bool) -> Self { + Self { + last_attempt: config_present.then_some(now), + ..Self::default() + } + } + + /// Decide whether to refresh now. Pure: same state plus same observation + /// always yields the same answer. + fn decide(&mut self, obs: Observation) -> Option<Refresh> { + if !obs.config_present { + // No config means no interface, so there is no handshake to age. + self.handshake_missing_since = None; + let Some(last) = self.last_attempt else { + return Some(MISSING_CONFIG); + }; + let delay = MISSING_CONFIG_RETRY_INTERVAL.max(self.backoff.delay()); + return (obs.now.saturating_sub(last) >= delay).then_some(MISSING_CONFIG); + } + + // Staleness is checked BEFORE the periodic refresh, and the + // missing-handshake timer is cleared ONLY by an observed handshake -- + // never by a refresh. HANDSHAKE_TIMEOUT equals REFRESH_INTERVAL, so if + // a periodic refresh reset the timer it would re-arm at 0 while the + // timer had only reached REFRESH_INTERVAL - POLL_INTERVAL, and the + // forced branch would be unreachable for a peer that never handshakes. + // That matters because gateway setup short-circuits on an unchanged + // config unless force is set: a CVM whose WireGuard config is correct + // but whose tunnel is dead can only recover through a forced refresh. + let silent_since = match obs.latest_handshake { + Some(handshake) => { + self.handshake_missing_since = None; + handshake + } + // No handshake yet. Time it from when we first noticed rather than + // from process start, so a freshly created interface gets a full + // HANDSHAKE_TIMEOUT to complete its first handshake. + None => *self.handshake_missing_since.get_or_insert(obs.now), + }; + if obs.now.saturating_sub(silent_since) >= HANDSHAKE_TIMEOUT { + // A forced refresh bounces the interface and re-requests + // certificates, so cap it at one attempt per HANDSHAKE_TIMEOUT. + // Unthrottled, a gateway that stays down would be hit on every + // poll: a self-inflicted flood aimed at something already broken. + let due = match self.last_force { + None => true, + Some(last) => obs.now.saturating_sub(last) >= HANDSHAKE_TIMEOUT, + }; + // Return either way. While the tunnel is dead a periodic refresh + // is pointless, since only the forced path can rebuild it. + return due.then_some(STALE_HANDSHAKE); + } + + // The first poll always re-registers: boot may have left the CVM + // unregistered, and re-registering a healthy CVM is cheap. + let Some(last) = self.last_attempt else { + return Some(PERIODIC); + }; + (obs.now.saturating_sub(last) >= REFRESH_INTERVAL).then_some(PERIODIC) + } + + /// Record the outcome of a refresh triggered by [`Checker::decide`]. + fn record(&mut self, now: i64, refresh: Refresh, succeeded: bool) { + self.last_attempt = Some(now); + if refresh.force { + self.last_force = Some(now); + } + // handshake_missing_since is deliberately NOT cleared here; see decide(). + self.backoff.record(succeeded); + } +} + +fn now_secs() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) +} + +/// Read the most recent handshake across all peers of the interface. +/// +/// `wg show <if> latest-handshakes` prints one `<pubkey>\t<unix-ts>` line per +/// peer, with `0` meaning "never". Returns `None` when the interface is absent, +/// has no peers, or no peer has ever completed a handshake — all of which the +/// caller treats the same way. +fn latest_handshake(interface: &str) -> Option<i64> { + let output = cmd!(wg show $interface latest-handshakes).ok()?; + parse_latest_handshake(&output) +} + +fn parse_latest_handshake(output: &str) -> Option<i64> { + output + .lines() + .filter_map(|line| line.split_whitespace().nth(1)) + .filter_map(|ts| ts.parse::<i64>().ok()) + .filter(|ts| *ts > 0) + .max() +} + +fn configured_gateway_interfaces() -> Vec<String> { + let Ok(entries) = std::fs::read_dir("/etc/wireguard") else { + return Vec::new(); + }; + entries + .filter_map(Result::ok) + .filter_map(|entry| entry.file_name().into_string().ok()) + .filter_map(|name| { + name.strip_suffix(".conf") + .filter(|name| name.starts_with("dstack-wg")) + .map(str::to_string) + }) + .collect() +} + +fn observe(now: i64) -> Observation { + let interfaces = configured_gateway_interfaces(); + let config_present = !interfaces.is_empty(); + let handshakes = interfaces + .iter() + .map(|interface| latest_handshake(interface)) + .collect::<Option<Vec<_>>>(); + Observation { + now, + config_present, + // Without a config the interface was never brought up, so there is no + // handshake to read and `decide` would not look at one anyway. Skipping + // the probe matters because that is precisely the state a gateway + // outage parks the CVM in: otherwise we would fork `wg` every poll for + // the entire outage to answer a question nobody asks. + // The least healthy cluster drives recovery. A missing handshake on + // any configured interface is represented as None; otherwise the + // oldest cluster handshake is the staleness boundary. + latest_handshake: handshakes.and_then(|values| values.into_iter().min()), + } +} + +/// systemd liveness reporting, inert when the unit has no `WatchdogSec`. +/// +/// The recovery paths in this loop only work while the loop runs, and nothing +/// below it can guarantee that: a wedged refresh leaves a healthy-looking +/// process that systemd will never restart. Handing liveness to systemd covers +/// a hang wherever it comes from, including causes not anticipated here. +struct Watchdog { + enabled: bool, +} + +impl Watchdog { + /// Report readiness and arm the watchdog. Readiness is sent before the + /// checker decides whether it has anything to do, so the paths that exit + /// straight away are still a started service that then stopped, not a + /// service that failed to start. + fn arm() -> Self { + let mut usec = 0; + let enabled = sd_notify::watchdog_enabled(false, &mut usec); + if let Err(error) = sd_notify::notify(false, &[NotifyState::Ready]) { + warn!("failed to report readiness to systemd: {error}"); + } + if enabled { + info!("systemd watchdog armed, timeout={usec}us"); + } + Self { enabled } + } + + fn ping(&self) { + if !self.enabled { + return; + } + if let Err(error) = sd_notify::notify(false, &[NotifyState::Watchdog]) { + warn!("failed to ping the systemd watchdog: {error}"); + } + } +} + +pub async fn cmd_gateway_checker(args: GatewayCheckerArgs) -> Result<()> { + let watchdog = Watchdog::arm(); + let refresher = + GatewayRefresher::load(&args.work_dir).context("failed to load gateway configuration")?; + + // Nothing to supervise for an app that never asked for a gateway. Exit + // successfully instead of polling forever; the unit is Restart=on-failure + // so systemd leaves the service alone. gateway_enabled is fixed by the + // measured app-compose and cannot change without a reboot. + if !refresher.gateway_enabled() { + info!("dstack-gateway is not enabled; nothing to check"); + return Ok(()); + } + + // A missing app id or gateway URL is a deployment mistake, not an outage. + // Both come from data fixed for the lifetime of the VM (app keys and the + // host-shared copy taken at setup), so no amount of retrying can fix it. + // Returning a plain error would have systemd restart us every RestartSec + // forever, so exit with the code the unit pins in RestartPreventExitStatus: + // that stops the respawn while still leaving the unit in `failed` state, + // which is what makes the mistake visible to the operator. + if let Err(error) = refresher.check_config() { + error!("dstack-gateway is enabled but misconfigured: {error:#}"); + error!("not retrying; this cannot be fixed without redeploying the CVM"); + std::process::exit(EXIT_MISCONFIGURED); + } + + info!("watching dstack-gateway registration"); + // The checker does not report gateway state to the host. Boot already + // reports the one signal that matters -- this CVM came up without a route + // -- and mirroring every later transition would mean tracking what the host + // has been told, which is state this loop should not have to carry. The + // consequence is that a boot error stays on the VMM after the checker + // recovers, until the VM restarts. + let mut checker = Checker::starting(now_secs(), !configured_gateway_interfaces().is_empty()); + loop { + // Ping before the work, not after, so a refresh that never returns + // stops the pings. Nothing else can do this for us: a refresh spends + // most of its time in blocking `cmd!` shell-outs (`wg-quick up` alone + // resolves peer endpoints), which occupy a runtime worker with no await + // point. tokio::time::timeout cannot cancel that, and a watchdog task + // on another worker would happily keep pinging while this loop is + // wedged. Only the loop itself can prove the loop is alive. + watchdog.ping(); + + let now = now_secs(); + if let Some(refresh) = checker.decide(observe(now)) { + info!("refreshing dstack-gateway: {}", refresh.reason); + let succeeded = match refresher.refresh(refresh.force).await { + Ok(()) => { + info!("dstack-gateway refresh succeeded"); + true + } + Err(error) => { + warn!("dstack-gateway refresh failed: {error:#}"); + false + } + }; + // now_secs() is re-read here rather than reusing `now`: a refresh can + // block on network timeouts for a long time, and the backoff has to + // count from when the attempt ended, not when it started. + checker.record(now_secs(), refresh, succeeded); + } + tokio::time::sleep(POLL_INTERVAL).await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const T0: i64 = 1_700_000_000; + + fn obs(now_offset: i64, config_present: bool, latest_handshake: Option<i64>) -> Observation { + Observation { + now: T0 + now_offset, + config_present, + latest_handshake, + } + } + + /// Replay the checker against a scripted world for `duration` seconds at the + /// production poll interval, returning (forced, periodic) refresh counts. + /// + /// `handshake` maps elapsed seconds to what `wg show ... latest-handshakes` + /// would report at that moment. Every refresh is treated as succeeding, so + /// the counts isolate the decision logic from the backoff. + /// + /// Deliberately starts from `Checker::default()`, not `Checker::starting`: + /// the upstream harness began with `LAST_REFRESH=0`, so this reproduces its + /// cadence exactly. The startup grace period is covered separately. + fn replay(duration: i64, handshake: impl Fn(i64) -> Option<i64>) -> (usize, usize) { + let mut checker = Checker::default(); + let (mut forced, mut periodic) = (0, 0); + let mut elapsed = 0; + while elapsed < duration { + let observation = obs(elapsed, true, handshake(elapsed)); + if let Some(refresh) = checker.decide(observation) { + if refresh.force { + forced += 1; + } else { + periodic += 1; + } + checker.record(T0 + elapsed, refresh, true); + } + elapsed += POLL_INTERVAL.as_secs() as i64; + } + (forced, periodic) + } + + /// The scenario table from the upstream fix that made the forced branch + /// reachable and rate limited it (`fix(guest): let the missing-handshake + /// timer actually expire`). These counts are the contract that commit + /// established by replaying the shell checker against a mocked clock; the + /// Rust port has to reproduce them exactly or it has silently regressed. + #[test] + fn reproduces_the_upstream_forced_refresh_scenarios() { + // Healthy tunnel: handshakes keep landing, nothing is ever forced. + assert_eq!(replay(900, |t| Some(T0 + t)).0, 0, "healthy handshakes"); + + // A peer that never handshakes must still reach the forced path, once + // per HANDSHAKE_TIMEOUT: at 180, 360, 540 and 720 seconds. + assert_eq!(replay(900, |_| None).0, 4, "never handshakes"); + + // Frozen handshake with the gateway down: the timestamp never advances, + // so it ages past the timeout and is forced on the same cadence. Before + // the rate limit this was one forced refresh per poll. + assert_eq!( + replay(900, |_| Some(T0)).0, + 4, + "handshake frozen, gateway down" + ); + + // Handshakes land for the first 180s, then the peer goes silent. The + // timer runs from the last real handshake, so forcing starts at 360. + let (forced, _) = replay(900, |t| Some(T0 + if t < 180 { t } else { 180 })); + assert_eq!(forced, 3, "handshake then peer gone"); + + // No handshake, then the gateway recovers at 540s. Two forced attempts + // (180, 360) and then the periodic cadence resumes. + let (forced, periodic) = replay(900, |t| (t >= 540).then_some(T0 + t)); + assert_eq!(forced, 2, "no handshake, gateway recovers"); + assert!(periodic >= 1, "periodic refresh must resume after recovery"); + } + + /// The regression the upstream fix was about: a periodic refresh must not + /// reset the missing-handshake timer. HANDSHAKE_TIMEOUT == REFRESH_INTERVAL, + /// so clearing it on every periodic refresh re-arms the timer one poll + /// before it can expire and the forced branch becomes unreachable. + #[test] + fn periodic_refresh_does_not_re_arm_the_missing_handshake_timer() { + assert_eq!( + HANDSHAKE_TIMEOUT, REFRESH_INTERVAL, + "this regression only bites while the two intervals are equal" + ); + let mut checker = Checker::default(); + // First poll: no handshake yet, so the timer starts and we re-register. + assert_eq!(checker.decide(obs(0, true, None)), Some(PERIODIC)); + checker.record(T0, PERIODIC, true); + assert_eq!(checker.handshake_missing_since, Some(T0)); + + // Still no handshake one full timeout later. The forced branch must + // fire; if the refresh above had cleared the timer it would not. + assert_eq!( + checker.decide(obs(HANDSHAKE_TIMEOUT, true, None)), + Some(STALE_HANDSHAKE) + ); + assert_eq!( + checker.handshake_missing_since, + Some(T0), + "the timer must survive the refresh that ran at t=0" + ); + } + + #[test] + fn staleness_outranks_the_periodic_refresh() { + let mut checker = Checker::default(); + checker.record(T0, PERIODIC, true); + checker.handshake_missing_since = Some(T0); + // At exactly REFRESH_INTERVAL both deadlines are due. The forced path + // must win: only it can rebuild a tunnel whose config is unchanged. + assert_eq!( + checker.decide(obs(REFRESH_INTERVAL, true, None)), + Some(STALE_HANDSHAKE) + ); + } + + #[test] + fn forced_refresh_is_rate_limited_while_the_tunnel_stays_dead() { + let mut checker = Checker::default(); + checker.record(T0, PERIODIC, true); + checker.handshake_missing_since = Some(T0); + + let refresh = checker.decide(obs(180, true, None)).expect("forced"); + assert!(refresh.force); + checker.record(T0 + 180, refresh, true); + + // Every poll in the next window is still stale, but must stay quiet -- + // and must not fall through to a periodic refresh either. + let mut elapsed = 190; + while elapsed < 360 { + assert_eq!( + checker.decide(obs(elapsed, true, None)), + None, + "forced refresh must not repeat at t={elapsed}" + ); + elapsed += 10; + } + assert_eq!( + checker.decide(obs(360, true, None)), + Some(STALE_HANDSHAKE), + "one forced attempt per handshake timeout" + ); + } + + #[test] + fn an_observed_handshake_is_the_only_thing_that_clears_the_timer() { + let mut checker = Checker::default(); + checker.record(T0, PERIODIC, true); + + assert!(checker.decide(obs(10, true, None)).is_none()); + assert_eq!(checker.handshake_missing_since, Some(T0 + 10)); + // A handshake lands. + assert!(checker.decide(obs(20, true, Some(T0 + 20))).is_none()); + assert_eq!(checker.handshake_missing_since, None); + // Losing it again restarts the clock rather than firing immediately. + assert!(checker.decide(obs(30, true, None)).is_none()); + assert_eq!(checker.handshake_missing_since, Some(T0 + 30)); + } + + #[test] + fn parses_max_handshake_across_peers() { + let output = "aaa\t1700000000\nbbb\t1700000042\nccc\t0\n"; + assert_eq!(parse_latest_handshake(output), Some(1_700_000_042)); + } + + #[test] + fn treats_never_handshaked_peers_as_no_handshake() { + assert_eq!(parse_latest_handshake("aaa\t0\nbbb\t0\n"), None); + assert_eq!(parse_latest_handshake(""), None); + assert_eq!(parse_latest_handshake("garbage\n"), None); + } + + #[test] + fn acts_immediately_when_boot_left_no_config() { + let mut checker = Checker::starting(T0, false); + assert_eq!(checker.decide(obs(0, false, None)), Some(MISSING_CONFIG)); + } + + /// Boot registers the CVM, then this checker starts. Re-registering right + /// away would double the gateway's registration load on every fleet reboot + /// for no gain, so a config that is already on disk starts the periodic + /// clock rather than triggering an immediate refresh. + #[test] + fn does_not_re_register_a_cvm_boot_just_registered() { + let mut checker = Checker::starting(T0, true); + assert_eq!(checker.decide(obs(0, true, Some(T0))), None); + assert_eq!(checker.decide(obs(170, true, Some(T0 + 170))), None); + assert_eq!( + checker.decide(obs(180, true, Some(T0 + 180))), + Some(PERIODIC), + "the periodic clock still runs from process start" + ); + } + + /// A checker that comes up with no config must not inherit the grace period + /// above: that state means boot's registration failed and the CVM has no + /// route at all. + #[test] + fn a_missing_config_overrides_the_startup_grace_period() { + let mut checker = Checker::starting(T0, false); + assert_eq!(checker.decide(obs(0, false, None)), Some(MISSING_CONFIG)); + checker.record(T0, MISSING_CONFIG, true); + // Once it lands, the periodic clock takes over from the refresh. + assert_eq!(checker.decide(obs(10, true, Some(T0 + 10))), None); + assert_eq!( + checker.decide(obs(180, true, Some(T0 + 180))), + Some(PERIODIC) + ); + } + + #[test] + fn retries_missing_config_with_backoff_up_to_the_cap() { + let mut checker = Checker::default(); + let mut t = 0; + // First attempt fires immediately and fails. + assert_eq!(checker.decide(obs(t, false, None)), Some(MISSING_CONFIG)); + checker.record(T0 + t, MISSING_CONFIG, false); + + // 30s base delay, then 60s, then capped at MAX_RETRY_INTERVAL. + for expected_delay in [30, 60, 120, 120] { + for early in [10, expected_delay - 10] { + assert_eq!( + checker.decide(obs(t + early, false, None)), + None, + "must not retry after {early}s while waiting {expected_delay}s" + ); + } + t += expected_delay; + assert_eq!( + checker.decide(obs(t, false, None)), + Some(MISSING_CONFIG), + "must retry once {expected_delay}s have passed" + ); + checker.record(T0 + t, MISSING_CONFIG, false); + } + } + + #[test] + fn backoff_resets_after_a_success() { + let mut backoff = Backoff::default(); + assert_eq!(backoff.delay(), 0); + backoff.record(false); + assert_eq!(backoff.delay(), 30); + backoff.record(false); + assert_eq!(backoff.delay(), 60); + backoff.record(true); + assert_eq!(backoff.delay(), 0); + } + + #[test] + fn backoff_saturates_instead_of_overflowing() { + let backoff = Backoff { + consecutive_failures: u32::MAX, + }; + assert_eq!(backoff.delay(), MAX_RETRY_INTERVAL); + } + + #[test] + fn re_registers_periodically_while_healthy() { + let mut checker = Checker::default(); + checker.record(T0, PERIODIC, true); + + assert_eq!(checker.decide(obs(170, true, Some(T0 + 170))), None); + assert_eq!( + checker.decide(obs(180, true, Some(T0 + 180))), + Some(PERIODIC), + "periodic refresh must not disrupt a healthy tunnel" + ); + } + + #[test] + fn gives_a_new_interface_a_full_timeout_to_handshake() { + let mut checker = Checker::default(); + checker.record(T0, PERIODIC, true); + + // The timer runs from t=10, when the missing handshake was first + // observed, not from the refresh at t=0. + assert_eq!(checker.decide(obs(10, true, None)), None); + assert_eq!(checker.decide(obs(170, true, None)), None); + // At t=180 the timer has only reached 170, so the periodic refresh is + // what comes due. It must not disturb the timer. + assert_eq!(checker.decide(obs(180, true, None)), Some(PERIODIC)); + checker.record(T0 + 180, PERIODIC, true); + assert_eq!(checker.handshake_missing_since, Some(T0 + 10)); + // One poll later the timer finally expires and forces a refresh. + assert_eq!(checker.decide(obs(190, true, None)), Some(STALE_HANDSHAKE)); + } +} diff --git a/dstack/dstack-util/src/gpu_info.rs b/dstack/dstack-util/src/gpu_info.rs new file mode 100644 index 000000000..ea634b16d --- /dev/null +++ b/dstack/dstack-util/src/gpu_info.rs @@ -0,0 +1,282 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! One-shot NVML sampler behind `dstack-util gpu-info`. +//! +//! NVML calls cannot be cancelled: a driver that wedges during a GPU reset or a +//! CC handshake blocks the calling thread until it decides to return. Running +//! the sample in a short-lived process lets `dstack-guest-agent` kill it and +//! move on, and keeps `libnvidia-ml` out of the long-lived agent entirely. +//! +//! One process per sample also means `Nvml::init()` runs fresh every time, so a +//! driver that loads after the agent started is picked up on the next sample +//! instead of being cached as "no GPU" for the agent's lifetime. +//! +//! This command owns collection only. Caching, presence gating and timeouts +//! belong to the caller. + +use std::collections::HashSet; + +use anyhow::Result; +use guest_api::{GpuDevice, GpuInfoResponse}; +use nvml_wrapper::enum_wrappers::device::TemperatureSensor; +use nvml_wrapper::error::NvmlError; +use nvml_wrapper::{Device, Nvml}; +use tracing::{debug, warn}; + +/// Per-run state for error log deduplication. +/// +/// A single run touches each (device, field) pair once, so this only collapses +/// the repeats that a multi-GPU box would otherwise produce for the same +/// unsupported counter on every card. +#[derive(Default)] +struct Sampler { + warned: HashSet<(u32, &'static str)>, +} + +/// Samples every NVIDIA GPU visible to NVML and prints the result as one JSON +/// object on stdout. +/// +/// Exits zero even when NVML is unavailable: "no driver" is a result the caller +/// needs to record, not a failure of this command. A non-zero exit is reserved +/// for not being able to produce a result at all. +pub fn cmd_gpu_info() -> Result<()> { + let response = collect(); + serde_json::to_writer(std::io::stdout(), &response)?; + println!(); + Ok(()) +} + +fn collect() -> GpuInfoResponse { + let nvml = match Nvml::init() { + Ok(nvml) => nvml, + Err(err) => { + // Expected on any guest without an NVIDIA driver. The caller gates + // on PCI presence, so reaching here means a card is attached but + // the library is missing -- worth a warning. + warn!("failed to initialize NVML: {err}"); + return unavailable(format!("failed to initialize NVML: {err}")); + } + }; + sample(&mut Sampler::default(), &nvml) +} + +fn unavailable(error: impl Into<String>) -> GpuInfoResponse { + GpuInfoResponse { + gpus: vec![], + error: error.into(), + cc_ready: None, + cc_enabled: None, + sample_age_ms: None, + } +} + +fn sample(sampler: &mut Sampler, nvml: &Nvml) -> GpuInfoResponse { + let count = match nvml.device_count() { + Ok(count) => count, + Err(e) => { + let error = format!("failed to get NVML GPU count: {e}"); + warn!("{error}"); + return unavailable(error); + } + }; + + // Both CC queries are system-wide despite hanging off `Device`, so ask once + // through device 0 rather than once per card. + let (cc_ready, cc_enabled) = query_cc_state(sampler, nvml, count); + GpuInfoResponse { + gpus: (0..count) + .map(|index| collect_device(sampler, nvml, index)) + .collect(), + error: String::new(), + cc_ready, + cc_enabled, + sample_age_ms: None, + } +} + +/// Reads the system-wide confidential-computing state. +/// +/// `get_confidential_compute_state` is `nvmlSystemGetConfComputeGpusReadyState` +/// and `is_cc_enabled` is `nvmlSystemGetConfComputeSettings`; neither uses the +/// device handle it is called on. Any device will do, so device 0 is used. +fn query_cc_state(sampler: &mut Sampler, nvml: &Nvml, count: u32) -> (Option<bool>, Option<bool>) { + if count == 0 { + return (None, None); + } + let device = match nvml.device_by_index(0) { + Ok(device) => device, + Err(e) => { + log_query_error(sampler, 0, "cc_state", &e); + return (None, None); + } + }; + let ready = match device.get_confidential_compute_state() { + Ok(ready) => Some(ready), + Err(e) => { + log_query_error(sampler, 0, "cc_ready", &e); + None + } + }; + let enabled = match device.is_cc_enabled() { + Ok(enabled) => Some(enabled), + Err(e) => { + log_query_error(sampler, 0, "cc_enabled", &e); + None + } + }; + (ready, enabled) +} + +fn collect_device(sampler: &mut Sampler, nvml: &Nvml, index: u32) -> GpuDevice { + match nvml.device_by_index(index) { + Ok(device) => collect_device_fields(sampler, index, &device), + Err(e) => { + log_query_error(sampler, index, "device", &e); + GpuDevice { + index, + errors: vec![format!("device: {e}")], + ..Default::default() + } + } + } +} + +fn collect_device_fields(sampler: &mut Sampler, index: u32, device: &Device<'_>) -> GpuDevice { + let mut errors = Vec::new(); + + let uuid = match device.uuid() { + Ok(uuid) => uuid, + Err(e) => { + push_error(sampler, index, "uuid", e, &mut errors); + String::new() + } + }; + let pci_bus_id = match device.pci_info() { + Ok(pci) => pci.bus_id, + Err(e) => { + push_error(sampler, index, "pci_bus_id", e, &mut errors); + String::new() + } + }; + + let (utilization_gpu, utilization_memory) = match device.utilization_rates() { + Ok(util) => (Some(util.gpu), Some(util.memory)), + Err(e) => { + push_error(sampler, index, "utilization", e, &mut errors); + (None, None) + } + }; + + let (memory_total_bytes, memory_used_bytes, memory_free_bytes) = match device.memory_info() { + Ok(mem) => (Some(mem.total), Some(mem.used), Some(mem.free)), + Err(e) => { + push_error(sampler, index, "memory", e, &mut errors); + (None, None, None) + } + }; + + let temperature_c = match device.temperature(TemperatureSensor::Gpu) { + Ok(temp) => Some(temp), + Err(e) => { + push_error(sampler, index, "temperature", e, &mut errors); + None + } + }; + + let power_usage_mw = match device.power_usage() { + Ok(power) => Some(power), + Err(e) => { + push_error(sampler, index, "power", e, &mut errors); + None + } + }; + + GpuDevice { + index, + uuid, + pci_bus_id, + utilization_gpu, + utilization_memory, + memory_total_bytes, + memory_used_bytes, + memory_free_bytes, + temperature_c, + power_usage_mw, + errors, + } +} + +fn push_error( + sampler: &mut Sampler, + index: u32, + field: &'static str, + err: NvmlError, + errors: &mut Vec<String>, +) { + errors.push(format!("{field}: {err}")); + log_query_error(sampler, index, field, &err); +} + +fn log_query_error(sampler: &mut Sampler, index: u32, field: &'static str, err: &NvmlError) { + let first = sampler.warned.insert((index, field)); + if matches!(err, NvmlError::NotSupported) { + // CC mode disables some counters by design; that is not an incident. + debug!("GPU {index} {field} not supported: {err}"); + } else if first { + warn!("failed to query GPU {index} {field}: {err}"); + } else { + debug!("failed to query GPU {index} {field}: {err}"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// NVML may or may not be present on the machine running this test. + /// Unavailability must be an error with no devices, not a silent empty + /// success (which would mean "NVML worked, zero GPUs"). + #[test] + fn collect_reports_nvml_unavailability_without_panicking() { + let info = collect(); + if info.error.is_empty() { + return; + } + assert!(info.gpus.is_empty()); + assert_eq!(info.cc_ready, None); + assert_eq!(info.cc_enabled, None); + } + + /// The guest agent parses stdout as one JSON object. A response must + /// survive that round trip with `None` distinct from `Some(0)`. + #[test] + fn response_round_trips_through_json() { + let original = GpuInfoResponse { + gpus: vec![GpuDevice { + index: 0, + uuid: "GPU-abc".into(), + pci_bus_id: "00000000:01:00.0".into(), + utilization_gpu: Some(0), + utilization_memory: None, + memory_total_bytes: Some(8 << 30), + memory_used_bytes: Some(0), + memory_free_bytes: None, + temperature_c: Some(40), + power_usage_mw: None, + errors: vec!["power: not supported".into()], + }], + error: String::new(), + cc_ready: Some(true), + cc_enabled: Some(true), + sample_age_ms: None, + }; + let encoded = serde_json::to_string(&original).expect("encode"); + assert!(!encoded.contains('\n'), "one JSON line per sample"); + let decoded: GpuInfoResponse = serde_json::from_str(&encoded).expect("decode"); + assert_eq!(decoded, original); + assert_eq!(decoded.gpus[0].utilization_gpu, Some(0)); + assert_eq!(decoded.gpus[0].utilization_memory, None); + } +} diff --git a/dstack/dstack-util/src/host_api.rs b/dstack/dstack-util/src/host_api.rs index 14715837e..e038ea3b2 100644 --- a/dstack/dstack-util/src/host_api.rs +++ b/dstack/dstack-util/src/host_api.rs @@ -17,6 +17,9 @@ use ra_tls::attestation::validate_tcb; use sodiumbox::{generate_keypair, open_sealed_box, PUBLICKEYBYTES}; use tracing::warn; +const HOST_API_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); +const PCCS_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + pub(crate) struct KeyProvision { pub sk: [u8; 32], pub mr: [u8; 32], @@ -47,7 +50,8 @@ impl HostApi { None => { let local_config: SysConfig = deserialize_json_file(format!("{HOST_SHARED_DIR}/{SYS_CONFIG}"))?; - Self::new(local_config.host_api_url, local_config.pccs_url) + let pccs = local_config.collateral_urls().pccs; + Self::new(local_config.host_api_url, pccs) } }; Ok(api) @@ -64,12 +68,15 @@ impl HostApi { let Some(client) = &self.client else { return Ok(()); }; - client - .notify(Notification { + tokio::time::timeout( + HOST_API_TIMEOUT, + client.notify(Notification { event: event.to_string(), payload: payload.to_string(), - }) - .await?; + }), + ) + .await + .context("Timed out notifying Host API")??; Ok(()) } @@ -87,12 +94,15 @@ impl HostApi { let Some(client) = &self.client else { return Err(anyhow!("Host API client not initialized")); }; - let provision = client - .get_sealing_key(host_api::GetSealingKeyRequest { + let provision = tokio::time::timeout( + HOST_API_TIMEOUT, + client.get_sealing_key(host_api::GetSealingKeyRequest { quote: quote.to_vec(), - }) - .await - .map_err(|err| anyhow!("Failed to get sealing key: {err:?}"))?; + }), + ) + .await + .context("Timed out requesting sealing key from Host API")? + .map_err(|err| anyhow!("Failed to get sealing key: {err:?}"))?; // verify the key provider quote let pccs_url = self @@ -101,10 +111,14 @@ impl HostApi { .map(str::trim) .filter(|url| !url.is_empty()) .unwrap_or(PHALA_PCCS_URL); - let verified_report = CollateralClient::with_default_http(pccs_url)? - .fetch_and_verify(&provision.provider_quote) - .await - .context("Failed to get quote collateral")?; + let collateral_client = CollateralClient::with_default_http(pccs_url)?; + let verified_report = tokio::time::timeout( + PCCS_TIMEOUT, + collateral_client.fetch_and_verify(&provision.provider_quote), + ) + .await + .context("Timed out fetching sealing-key quote collateral")? + .context("Failed to get quote collateral")?; validate_tcb(&verified_report)?; let sgx_report = verified_report .report diff --git a/dstack/dstack-util/src/host_shared.rs b/dstack/dstack-util/src/host_shared.rs new file mode 100644 index 000000000..c3495e78b --- /dev/null +++ b/dstack/dstack-util/src/host_shared.rs @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// SPDX-License-Identifier: Apache-2.0 + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result}; +use clap::{Parser, Subcommand}; +use dstack_types::shared_filenames::HOST_SHARED_DISK_LABEL; +use fs_err as fs; +use tracing::{info, warn}; + +#[derive(Parser)] +pub struct HostSharedArgs { + #[command(subcommand)] + pub command: HostSharedCommand, +} + +#[derive(Subcommand)] +pub enum HostSharedCommand { + /// Mount the host-provided shared directory read-only. + Mount(MountHostSharedArgs), + /// Unmount a host-provided shared directory. + Unmount(UnmountHostSharedArgs), +} + +#[derive(Parser)] +pub struct MountHostSharedArgs { + /// Directory where the host share is mounted. + #[arg(long)] + pub mount_point: PathBuf, +} + +#[derive(Parser)] +pub struct UnmountHostSharedArgs { + /// Mounted host-share directory. + #[arg(long)] + pub mount_point: PathBuf, +} + +fn find_disk_by_label(label: &str) -> Option<PathBuf> { + let label_path = PathBuf::from(format!("/dev/disk/by-label/{label}")); + if label_path.exists() { + return Some(label_path); + } + + let entries = fs::read_dir("/sys/block").ok()?; + for entry in entries.flatten() { + let dev_path = PathBuf::from("/dev").join(entry.file_name()); + let output = Command::new("blkid") + .args(["-s", "LABEL", "-o", "value"]) + .arg(&dev_path) + .output(); + if let Ok(output) = output { + if output.status.success() && String::from_utf8_lossy(&output.stdout).trim() == label { + return Some(dev_path); + } + } + } + None +} + +pub fn mount_host_shared(mount_point: &Path) -> Result<()> { + fs::create_dir_all(mount_point) + .with_context(|| format!("failed to create {}", mount_point.display()))?; + + if let Some(device) = find_disk_by_label(HOST_SHARED_DISK_LABEL) { + info!(device = %device.display(), "found host-shared disk"); + let status = Command::new("mount") + .args(["-o", "ro"]) + .arg(&device) + .arg(mount_point) + .status() + .with_context(|| format!("failed to run mount for {}", device.display()))?; + if status.success() { + info!(mount_point = %mount_point.display(), "mounted host-shared disk"); + return Ok(()); + } + warn!( + device = %device.display(), + status = %status, + "failed to mount host-shared disk, falling back to 9p" + ); + } else { + info!("host-shared disk not found, trying 9p"); + } + + let status = Command::new("mount") + .args([ + "-t", + "9p", + "-o", + "trans=virtio,version=9p2000.L,ro", + "host-shared", + ]) + .arg(mount_point) + .status() + .context("failed to run 9p mount")?; + anyhow::ensure!( + status.success(), + "failed to mount host-shared at {}", + mount_point.display() + ); + info!(mount_point = %mount_point.display(), "mounted host-shared via 9p"); + Ok(()) +} + +pub fn unmount_host_shared(mount_point: &Path) -> Result<()> { + let status = Command::new("umount") + .arg(mount_point) + .status() + .context("failed to run umount")?; + anyhow::ensure!( + status.success(), + "failed to unmount host-shared at {}", + mount_point.display() + ); + Ok(()) +} + +pub fn cmd_host_shared(args: HostSharedArgs) -> Result<()> { + match args.command { + HostSharedCommand::Mount(args) => mount_host_shared(&args.mount_point), + HostSharedCommand::Unmount(args) => unmount_host_shared(&args.mount_point), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_mount_command() { + let args = HostSharedArgs::try_parse_from([ + "host-shared", + "mount", + "--mount-point", + "/run/dstack/host-shared", + ]) + .unwrap(); + let HostSharedCommand::Mount(args) = args.command else { + panic!("expected mount command"); + }; + assert_eq!(args.mount_point, Path::new("/run/dstack/host-shared")); + } + + #[test] + fn parses_unmount_command() { + let args = HostSharedArgs::try_parse_from([ + "host-shared", + "unmount", + "--mount-point", + "/run/dstack/host-shared", + ]) + .unwrap(); + let HostSharedCommand::Unmount(args) = args.command else { + panic!("expected unmount command"); + }; + assert_eq!(args.mount_point, Path::new("/run/dstack/host-shared")); + } +} diff --git a/dstack/dstack-util/src/main.rs b/dstack/dstack-util/src/main.rs index dcaf358d3..10373f94d 100644 --- a/dstack/dstack-util/src/main.rs +++ b/dstack/dstack-util/src/main.rs @@ -7,6 +7,7 @@ use clap::{Parser, Subcommand}; use dstack_attest::emit_runtime_event; use dstack_types::{KeyProvider, KeyProviderKind}; use fs_err as fs; +use gateway_checker::{cmd_gateway_checker, GatewayCheckerArgs}; use getrandom::fill as getrandom; use host_api::HostApi; use k256::schnorr::SigningKey; @@ -17,6 +18,7 @@ use ra_tls::{ kdf::{derive_key, derive_p256_key_pair_from_bytes}, rcgen::KeyPair, }; +use safe_write::{safe_write, safe_write_with_mode}; use scale::Encode; use std::path::Path; use std::{ @@ -29,7 +31,10 @@ use utils::AppKeys; mod crypto; mod docker_compose; +mod gateway_checker; +mod gpu_info; mod host_api; +mod host_shared; mod parse_env_file; mod system_setup; mod utils; @@ -66,8 +71,12 @@ enum Commands { Rand(RandArgs), /// Prepare dstack system. Setup(SetupArgs), + /// Mount or unmount the host-provided shared directory. + HostShared(host_shared::HostSharedArgs), /// Refresh the dstack gateway configuration GatewayRefresh(GatewayRefreshArgs), + /// Keep the dstack gateway registration fresh (long-running) + GatewayChecker(GatewayCheckerArgs), /// Notify the host about the dstack app NotifyHost(HostNotifyArgs), /// Remove orphaned containers @@ -89,6 +98,12 @@ enum Commands { AttestStrip(AttestStripArgs), /// Get app keys from a KMS server GetKeys(GetKeysArgs), + /// Decrypt data encrypted with the app's environment encryption public key + Decrypt(DecryptArgs), + /// Encrypt data for an app using its KMS-provided environment encryption key + Encrypt(EncryptArgs), + /// Sample NVIDIA GPU telemetry through NVML and print it as JSON + GpuInfo, } #[derive(Parser)] @@ -358,6 +373,62 @@ struct GetKeysArgs { root_ca: Option<PathBuf>, } +#[derive(Parser)] +/// Decrypt data encrypted with the app's environment encryption public key +struct DecryptArgs { + /// Input file (default: stdin) + #[arg(short, long)] + input: Option<PathBuf>, + + /// Output file (default: stdout) + #[arg(short, long)] + output: Option<PathBuf>, + + /// App keys file containing env_crypt_key + #[arg(long)] + key_file: Option<PathBuf>, + + /// Decode the input as hexadecimal text before decrypting + #[arg(long)] + hex: bool, +} + +#[derive(Parser)] +/// Encrypt data for an app using its KMS-provided environment encryption key +struct EncryptArgs { + /// KMS server URL + #[arg(short, long)] + kms_url: String, + + /// Application ID (20 bytes in hex) + #[arg(long)] + app_id: String, + + /// Input file (default: stdin) + #[arg(short, long)] + input: Option<PathBuf>, + + /// Output file (default: stdout) + #[arg(short, long)] + output: Option<PathBuf>, + + /// Plaintext bytes per independently authenticated chunk + #[arg(long, default_value_t = crypto::DEFAULT_CHUNK_SIZE)] + chunk_size: usize, + + /// Root CA certificate (PEM format) used to verify the KMS TLS certificate + #[arg(long)] + root_ca: Option<PathBuf>, + + /// Trusted compressed secp256k1 KMS signer public key (hex) + #[arg(long)] + kms_pubkey: String, + + /// Maximum accepted age of the KMS public-key signature in seconds + #[arg(long, default_value_t = 300)] + max_signature_age: u64, +} + fn pad64(data: &[u8]) -> Result<[u8; 64]> { if data.len() > 64 { anyhow::bail!("report_data must be at most 64 bytes"); @@ -379,7 +450,11 @@ fn cmd_quote_report(args: QuoteReportArgs) -> Result<()> { } None => [0u8; 64], }; - let attestation = Attestation::quote(&report_data).context("Failed to get attestation")?; + if args.debug { + eprintln!("debug: quote diagnostics enabled; attestation policy is unchanged"); + } + let attestation = Attestation::quote_with_sys_config(&report_data, &args.sys_config) + .context("Failed to get attestation")?; let request = VerificationRequestJson { attestation: hex::encode(attestation.into_versioned().to_scale()?), }; @@ -387,7 +462,7 @@ fn cmd_quote_report(args: QuoteReportArgs) -> Result<()> { let json = serde_json::to_string_pretty(&request).context("Failed to serialize request JSON")?; if let Some(output_path) = args.output { - fs::write(&output_path, json).context("Failed to write quote report")?; + safe_write::safe_write(&output_path, json).context("Failed to write quote report")?; } else { println!("{json}"); } @@ -422,7 +497,7 @@ fn cmd_attest(args: AttestArgs) -> Result<()> { if args.hex { let encoded = hex::encode(&attestation); if let Some(output) = args.output { - fs::write(&output, encoded).context("Failed to write attestation hex")?; + safe_write::safe_write(&output, encoded).context("Failed to write attestation hex")?; } else { println!("{encoded}"); } @@ -432,7 +507,7 @@ fn cmd_attest(args: AttestArgs) -> Result<()> { let output = args .output .unwrap_or_else(|| PathBuf::from("attestation.bin")); - fs::write(&output, &attestation).context("Failed to write attestation sample")?; + safe_write::safe_write(&output, &attestation).context("Failed to write attestation sample")?; Ok(()) } @@ -450,7 +525,7 @@ fn cmd_attest_info(args: AttestInfoArgs) -> Result<()> { match attestation { VersionedAttestation::V0 { attestation } => { println!("version: V0"); - println!("mode: {:?}", attestation.quote.mode()); + println!("mode: {:?}", attestation.quote.variant()); println!("config_bytes: {}", attestation.config.len()); match attestation.tdx_quote() { Some(tdx) => { @@ -495,7 +570,7 @@ fn cmd_attest_json(args: AttestJsonArgs) -> Result<()> { let json = match attestation { VersionedAttestation::V0 { attestation } => { - let mode = attestation.quote.mode().as_str(); + let mode = attestation.quote.variant().as_str(); let tdx_quote = match attestation.tdx_quote() { Some(tdx) => serde_json::json!({ "quote": hex::encode(&tdx.quote), @@ -523,7 +598,7 @@ fn cmd_attest_json(args: AttestJsonArgs) -> Result<()> { let output = serde_json::to_string_pretty(&json).context("Failed to serialize JSON")?; if let Some(path) = args.output { - fs::write(&path, output).context("Failed to write JSON output")?; + safe_write::safe_write(&path, output).context("Failed to write JSON output")?; } else { println!("{output}"); } @@ -541,7 +616,8 @@ fn cmd_attest_strip(args: AttestStripArgs) -> Result<()> { let output = args .output .unwrap_or_else(|| PathBuf::from("attestation.strip.bin")); - fs::write(&output, stripped.to_scale()?).context("Failed to write stripped attestation")?; + safe_write::safe_write(&output, stripped.to_scale()?) + .context("Failed to write stripped attestation")?; Ok(()) } @@ -549,11 +625,7 @@ async fn cmd_get_keys(args: GetKeysArgs) -> Result<()> { use dstack_kms_rpc::kms_client::KmsClient; use ra_rpc::client::RaClientConfig; - let kms_url = if args.kms_url.ends_with("/prpc") { - args.kms_url.clone() - } else { - format!("{}/prpc", args.kms_url.trim_end_matches('/')) - }; + let kms_url = normalize_prpc_url(&args.kms_url); // Load root CA if provided for TLS pinning let root_ca_pem = if let Some(root_ca_path) = &args.root_ca { @@ -640,7 +712,7 @@ async fn cmd_get_keys(args: GetKeysArgs) -> Result<()> { // Step 5: Output result let json = serde_json::to_string_pretty(&keys).context("Failed to serialize app keys")?; if let Some(output_path) = args.output { - fs::write(&output_path, &json).context("Failed to write app keys")?; + safe_write_with_mode(&output_path, &json, 0o600).context("Failed to write app keys")?; eprintln!("App keys written to: {}", output_path.display()); } else { println!("{json}"); @@ -649,11 +721,239 @@ async fn cmd_get_keys(args: GetKeysArgs) -> Result<()> { Ok(()) } +fn cmd_decrypt(args: DecryptArgs) -> Result<()> { + use dstack_types::shared_filenames::{host_shared_dir, APP_KEYS}; + + let key_file = args + .key_file + .unwrap_or_else(|| host_shared_dir().join(APP_KEYS)); + let keys: AppKeys = utils::deserialize_json_file(&key_file) + .with_context(|| format!("failed to load app keys from {}", key_file.display()))?; + let env_crypt_key: [u8; 32] = keys + .env_crypt_key + .try_into() + .map_err(|key: Vec<u8>| anyhow::anyhow!("invalid env crypt key length: {}", key.len()))?; + + if args.hex { + let input = read_all_input(args.input.as_deref())?; + let input = decode_hex_ciphertext(&input)?; + return decrypt_auto( + env_crypt_key, + input.as_slice(), + open_output(args.output.as_deref())?, + ); + } + + let input = open_input(args.input.as_deref())?; + decrypt_auto(env_crypt_key, input, open_output(args.output.as_deref())?) +} + +fn decrypt_auto( + env_crypt_key: [u8; 32], + mut input: impl Read, + mut output: impl Write, +) -> Result<()> { + let mut prefix = Vec::with_capacity(crypto::STREAM_MAGIC.len()); + input + .by_ref() + .take(crypto::STREAM_MAGIC.len() as u64) + .read_to_end(&mut prefix) + .context("failed to read ciphertext")?; + if prefix == crypto::STREAM_MAGIC { + crypto::dh_decrypt_stream(env_crypt_key, input, output) + .context("failed to decrypt stream")?; + } else { + let mut ciphertext = prefix; + input + .read_to_end(&mut ciphertext) + .context("failed to read ciphertext")?; + let plaintext = crypto::dh_decrypt(env_crypt_key, &ciphertext) + .context("failed to decrypt legacy input")?; + output + .write_all(&plaintext) + .context("failed to write plaintext")?; + } + Ok(()) +} + +async fn cmd_encrypt(args: EncryptArgs) -> Result<()> { + use dstack_kms_rpc::kms_client::KmsClient; + use ra_rpc::client::RaClientConfig; + + let app_id = decode_app_id(Some(&args.app_id))?.context("app_id is required")?; + let kms_url = normalize_prpc_url(&args.kms_url); + let root_ca_pem = args + .root_ca + .as_ref() + .map(|path| { + fs::read_to_string(path) + .with_context(|| format!("failed to read root CA from {}", path.display())) + }) + .transpose()?; + let client = RaClientConfig::builder() + .remote_uri(kms_url) + .tls_no_check(false) + .tls_built_in_root_certs(root_ca_pem.is_none()) + .maybe_tls_ca_cert(root_ca_pem) + .build() + .into_client() + .context("failed to create KMS client")?; + let response = KmsClient::new(client) + .get_app_env_encrypt_pub_key(dstack_kms_rpc::AppId { + app_id: app_id.to_vec(), + }) + .await + .context("failed to get app environment encryption public key")?; + let public_key: [u8; 32] = response + .public_key + .try_into() + .map_err(|key: Vec<u8>| anyhow::anyhow!("invalid public key length: {}", key.len()))?; + verify_env_encrypt_public_key( + &public_key, + &response.signature_v1, + &app_id, + response.timestamp, + &args.kms_pubkey, + args.max_signature_age, + )?; + + crypto::dh_encrypt_stream( + public_key, + open_input(args.input.as_deref())?, + open_output(args.output.as_deref())?, + args.chunk_size, + ) + .context("failed to encrypt stream") +} + +fn normalize_prpc_url(url: &str) -> String { + let url = url.trim_end_matches('/'); + if url.ends_with("/prpc") { + url.to_string() + } else { + format!("{url}/prpc") + } +} + +fn decode_hex_ciphertext(input: &[u8]) -> Result<Vec<u8>> { + hex_decode( + std::str::from_utf8(input) + .context("hex ciphertext is not valid UTF-8")? + .trim(), + ) + .context("failed to decode hex ciphertext") +} + +fn verify_env_encrypt_public_key( + public_key: &[u8; 32], + signature: &[u8], + app_id: &[u8; 20], + timestamp: u64, + trusted_pubkey: &str, + max_age: u64, +) -> Result<()> { + use k256::ecdsa::{RecoveryId, Signature, VerifyingKey}; + use sha3::{Digest, Keccak256}; + use std::time::{SystemTime, UNIX_EPOCH}; + + const FUTURE_SKEW: u64 = 60; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system time is before the Unix epoch")? + .as_secs(); + anyhow::ensure!( + timestamp <= now.saturating_add(FUTURE_SKEW), + "kms public-key signature timestamp is too far in the future" + ); + anyhow::ensure!( + now.saturating_sub(timestamp) <= max_age, + "kms public-key signature is too old" + ); + anyhow::ensure!(signature.len() == 65, "invalid KMS signature length"); + + let signature_value = + Signature::from_slice(&signature[..64]).context("invalid KMS signature")?; + let recovery_id = RecoveryId::from_byte(signature[64]).context("invalid KMS recovery ID")?; + let digest = Keccak256::new_with_prefix( + [ + b"dstack-env-encrypt-pubkey".as_slice(), + b":".as_slice(), + app_id.as_slice(), + ×tamp.to_be_bytes(), + public_key.as_slice(), + ] + .concat(), + ); + let recovered = VerifyingKey::recover_from_digest(digest, &signature_value, recovery_id) + .context("failed to recover KMS signer public key")?; + + let trusted_pubkey = trusted_pubkey.strip_prefix("0x").unwrap_or(trusted_pubkey); + let trusted_pubkey = + hex_decode(trusted_pubkey).context("invalid trusted KMS public key hex")?; + let trusted = + VerifyingKey::from_sec1_bytes(&trusted_pubkey).context("invalid trusted KMS public key")?; + anyhow::ensure!( + recovered == trusted, + "kms public-key signature was made by an untrusted signer" + ); + Ok(()) +} + +fn read_all_input(path: Option<&Path>) -> Result<Vec<u8>> { + let mut input = open_input(path)?; + let mut data = Vec::new(); + input + .read_to_end(&mut data) + .context("failed to read input")?; + Ok(data) +} + +fn open_input(path: Option<&Path>) -> Result<Box<dyn Read>> { + match path { + Some(path) => { + Ok(Box::new(fs::File::open(path).with_context(|| { + format!("failed to open input {}", path.display()) + })?)) + } + None => Ok(Box::new(io::stdin())), + } +} + +fn open_output(path: Option<&Path>) -> Result<Box<dyn Write>> { + use fs_err::os::unix::fs::OpenOptionsExt; + use std::os::unix::fs::PermissionsExt; + + match path { + Some(path) => { + let file = fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o600) + .open(path) + .with_context(|| format!("failed to open output {}", path.display()))?; + file.set_permissions(std::fs::Permissions::from_mode(0o600)) + .with_context(|| format!("Failed to set permissions on {}", path.display()))?; + Ok(Box::new(file)) + } + None => Ok(Box::new(io::stdout())), + } +} + fn cmd_quote() -> Result<()> { - let mut report_data = [0; 64]; + let mut input = Vec::with_capacity(65); io::stdin() - .read_exact(&mut report_data) + .take(65) + .read_to_end(&mut input) .context("Failed to read report data")?; + anyhow::ensure!( + input.len() == 64, + "report data must be exactly 64 bytes (received {})", + input.len() + ); + let report_data: [u8; 64] = input + .try_into() + .map_err(|_| anyhow::anyhow!("invalid report data length"))?; // Platform-adaptive: detect the running TEE and emit its raw hardware quote // (the TDX DCAP quote, or the AMD SEV-SNP report). For a verifier-ready, // platform-agnostic payload (with event log / mr_config), use `quote-report`. @@ -695,26 +995,22 @@ fn cmd_rand(rand_args: RandArgs) -> Result<()> { if rand_args.hex { data = hex::encode(data).into_bytes(); } - io::stdout() - .write_all(&data) - .context("Failed to write random data")?; + if let Some(output) = rand_args.output { + // key material: owner-only, and never half-written — a truncated + // random file would pass for a valid secret. + safe_write::safe_write_with_mode(&output, &data, 0o600) + .with_context(|| format!("Failed to write random output {output}"))?; + } else { + io::stdout() + .write_all(&data) + .context("Failed to write random data")?; + } Ok(()) } fn cmd_show_mrs() -> Result<()> { - let attestation = match ra_tls::attestation::Attestation::local() { - Ok(attestation) => attestation, - Err(err) => { - // DEV-ONLY fallback: no TEE hardware present (e.g. a local no_tee - // dev CVM). There are no measurement registers to show, so skip - // instead of failing. Real TEE hosts always resolve an - // attestation above and print MRs as before. - tracing::warn!( - "no TEE attestation available ({err:#}); skipping MR display (no_tee dev mode)" - ); - return Ok(()); - } - }; + let attestation = + ra_tls::attestation::Attestation::local().context("Failed to get attestation")?; let app_info = attestation .into_v1() .decode_app_info(false) @@ -801,8 +1097,9 @@ fn cmd_gen_ra_cert(args: GenRaCertArgs) -> Result<()> { let ca_cert = fs::read_to_string(args.ca_cert)?; let ca_key = fs::read_to_string(args.ca_key)?; let cert_pair = generate_ra_cert(ca_cert, ca_key)?; - fs::write(&args.cert_path, cert_pair.cert_pem).context("Failed to write certificate")?; - fs::write(&args.key_path, cert_pair.key_pem).context("Failed to write private key")?; + safe_write(&args.cert_path, &cert_pair.cert_pem).context("Failed to write certificate")?; + safe_write_with_mode(&args.key_path, &cert_pair.key_pem, 0o600) + .context("Failed to write private key")?; Ok(()) } @@ -827,8 +1124,9 @@ fn cmd_gen_ca_cert(args: GenCaCertArgs) -> Result<()> { let cert = req .self_signed() .context("Failed to self-sign certificate")?; - fs::write(&args.cert, cert.pem()).context("Failed to write certificate")?; - fs::write(&args.key, key.serialize_pem()).context("Failed to write private key")?; + safe_write(&args.cert, cert.pem()).context("Failed to write certificate")?; + safe_write_with_mode(&args.key, key.serialize_pem(), 0o600) + .context("Failed to write private key")?; Ok(()) } @@ -843,7 +1141,7 @@ fn cmd_gen_app_keys(args: GenAppKeysArgs) -> Result<()> { }; let app_keys = make_app_keys(&key, &disk_key, &k256_key, args.ca_level, key_provider)?; let app_keys = serde_json::to_string(&app_keys).context("Failed to serialize app keys")?; - fs::write(&args.output, app_keys).context("Failed to write app keys")?; + safe_write_with_mode(&args.output, &app_keys, 0o600).context("Failed to write app keys")?; Ok(()) } @@ -885,26 +1183,12 @@ fn make_app_keys( use ra_tls::cert::CertRequest; let pubkey = app_key.public_key_der(); let report_data = QuoteContentType::RaTlsCert.to_report_data(&pubkey); - let attestation = match Attestation::quote(&report_data) { - Ok(attestation) => Some(attestation.into_versioned()), - Err(err) => { - // DEV-ONLY fallback: no TEE hardware present (e.g. a local no_tee - // dev CVM). Issue a plain self-signed cert with no ra-tls - // attestation extension instead of failing app-key generation. - // Real TEE hosts always resolve a quote above and take the - // unchanged path; the KMS key-provider path never calls this - // function locally (KMS keys come from the KMS server), so this - // only affects the None/Local key providers used by local, - // unattested dev CVMs. - tracing::warn!( - "no TEE attestation available ({err:#}); issuing app keys without an attestation (no_tee dev mode)" - ); - None - } - }; + let attestation = Attestation::quote(&report_data) + .context("Failed to get attestation")? + .into_versioned(); let req = CertRequest::builder() .subject("App Root Cert") - .maybe_attestation(attestation.as_ref()) + .attestation(&attestation) .key(app_key) .ca_level(ca_level) .build(); @@ -1144,10 +1428,12 @@ fn cmd_vtpm_attest(args: VtpmAttestArgs) -> Result<()> { if let Some(error) = &result.error { println!("Error: {}", error); } - anyhow::bail!("attestation failed"); } } + if !result.success { + anyhow::bail!("attestation failed"); + } Ok(()) } @@ -1189,7 +1475,8 @@ fn cmd_tpm_quote(args: TpmQuoteArgs) -> Result<()> { serde_json::to_string_pretty(&tpm_quote).context("Failed to serialize TPM quote")?; if let Some(output_path) = args.output { - fs::write(&output_path, quote_json).context("Failed to write quote to file")?; + safe_write_with_mode(&output_path, "e_json, 0o600) + .context("Failed to write quote to file")?; eprintln!("TPM quote written to: {:?}", output_path); } else { println!("{}", quote_json); @@ -1213,7 +1500,7 @@ async fn cmd_tpm_verify(args: TpmVerifyArgs) -> Result<()> { println!("[Step 1] Fetching quote collateral (certificates + CRLs)..."); let collateral = tpm_qvl::get_collateral(&tpm_quote, &root_ca_pem) .await - .context("Failed to get collateral")?; + .context("failed to get TPM collateral")?; let crl_count = collateral.crls.len() + if collateral.root_ca_crl.is_some() { 1 @@ -1283,14 +1570,20 @@ async fn cmd_tpm_verify(args: TpmVerifyArgs) -> Result<()> { #[tokio::main] async fn main() -> Result<()> { + let cli = Cli::parse(); { use tracing_subscriber::{fmt, EnvFilter}; let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); - fmt().with_env_filter(filter).with_ansi(false).init(); + let builder = fmt().with_env_filter(filter).with_ansi(false); + // `gpu-info` writes a machine-readable JSON document to stdout, so its + // logs must go to stderr or an NVML warning would corrupt the output. + // Every other subcommand keeps the historical stdout behaviour. + match cli.command { + Commands::GpuInfo => builder.with_writer(std::io::stderr).init(), + _ => builder.init(), + } } - let cli = Cli::parse(); - match cli.command { Commands::Quote => cmd_quote()?, Commands::Eventlog => cmd_eventlog()?, @@ -1317,6 +1610,10 @@ async fn main() -> Result<()> { Commands::Setup(args) => { cmd_sys_setup(args).await?; } + Commands::HostShared(args) => host_shared::cmd_host_shared(args)?, + Commands::GatewayChecker(args) => { + cmd_gateway_checker(args).await?; + } Commands::GatewayRefresh(args) => { cmd_gateway_refresh(args).await?; } @@ -1361,7 +1658,173 @@ async fn main() -> Result<()> { Commands::GetKeys(args) => { cmd_get_keys(args).await?; } + Commands::Decrypt(args) => { + cmd_decrypt(args)?; + } + Commands::Encrypt(args) => { + cmd_encrypt(args).await?; + } + Commands::GpuInfo => { + gpu_info::cmd_gpu_info()?; + } } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + fn rand_args(output: Option<String>, bytes: usize, hex: bool) -> RandArgs { + RandArgs { bytes, output, hex } + } + + /// `-o` used to be parsed and then ignored, so the file was never created + /// and the bytes went to stdout instead. + #[test] + fn rand_writes_to_the_requested_output_path() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("secret.bin"); + + cmd_rand(rand_args(Some(path.display().to_string()), 32, false)).unwrap(); + + assert_eq!(fs::metadata(&path).unwrap().len(), 32); + // nothing but the target: no temporary file left behind. + assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 1); + } + + /// The output is key material, so it must never be readable by anyone else. + #[test] + fn rand_output_is_owner_only() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("secret.bin"); + + cmd_rand(rand_args(Some(path.display().to_string()), 32, false)).unwrap(); + + let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600, "random output must be 0600, got {mode:o}"); + } + + #[test] + fn rand_hex_output_is_twice_as_long() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("secret.hex"); + + cmd_rand(rand_args(Some(path.display().to_string()), 16, true)).unwrap(); + + let body = fs::read(&path).unwrap(); + assert_eq!(body.len(), 32); + assert!(body.iter().all(|b| b.is_ascii_hexdigit())); + } + + /// Re-running must replace the file rather than failing, so a retry after a + /// partial or interrupted run cannot wedge the caller. + #[test] + fn rand_replaces_an_existing_output() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("secret.bin"); + + cmd_rand(rand_args(Some(path.display().to_string()), 8, false)).unwrap(); + let first = fs::read(&path).unwrap(); + + cmd_rand(rand_args(Some(path.display().to_string()), 32, false)).unwrap(); + let second = fs::read(&path).unwrap(); + + assert_eq!(first.len(), 8); + assert_eq!(second.len(), 32); + assert_eq!( + fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + + #[test] + fn prpc_url_normalization_handles_trailing_slashes() { + assert_eq!( + normalize_prpc_url("https://kms.example.com/prpc/"), + "https://kms.example.com/prpc" + ); + assert_eq!( + normalize_prpc_url("https://kms.example.com/"), + "https://kms.example.com/prpc" + ); + } + + #[test] + fn decrypt_auto_detects_stream_and_falls_back_to_legacy() { + use x25519_dalek::{PublicKey, StaticSecret}; + + let secret = StaticSecret::random_from_rng(rand::thread_rng()); + let mut encrypted = Vec::new(); + crypto::dh_encrypt_stream( + PublicKey::from(&secret).to_bytes(), + b"stream plaintext".as_slice(), + &mut encrypted, + 4, + ) + .unwrap(); + let mut decrypted = Vec::new(); + decrypt_auto(secret.to_bytes(), encrypted.as_slice(), &mut decrypted).unwrap(); + assert_eq!(decrypted, b"stream plaintext"); + + let legacy_secret: [u8; 32] = + hex_decode("7c282bf94b35dc47801dc953bfa0896fc2bd313381d3e8eca4e42f6536d2a96f") + .unwrap() + .try_into() + .unwrap(); + let legacy_ciphertext = hex_decode("0bd18749612f4c8b9dd583c7d6a646b90abd34e3c731a7708d0caf9039095641e1f0948e775f0b7351788db7f246d51806954626dcccb6a60d64665ca3715c6bef75616cab476d27bba04080361200d6a58cec").unwrap(); + let mut legacy_plaintext = Vec::new(); + decrypt_auto( + legacy_secret, + legacy_ciphertext.as_slice(), + &mut legacy_plaintext, + ) + .unwrap(); + assert_eq!(legacy_plaintext, b"[{\"key\":\"\",\"value\":\"\"}]"); + assert_eq!(decode_hex_ciphertext(b" 00ff\n").unwrap(), [0, 255]); + } + + #[test] + fn env_encrypt_public_key_requires_the_trusted_signer() { + use k256::ecdsa::SigningKey as EcdsaSigningKey; + use sha3::{Digest, Keccak256}; + use std::time::{SystemTime, UNIX_EPOCH}; + + let signer = EcdsaSigningKey::random(&mut rand::thread_rng()); + let app_id = [0x11; 20]; + let public_key = [0x22; 32]; + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + let digest = Keccak256::new_with_prefix( + [ + b"dstack-env-encrypt-pubkey".as_slice(), + b":".as_slice(), + app_id.as_slice(), + ×tamp.to_be_bytes(), + public_key.as_slice(), + ] + .concat(), + ); + let (signature, recovery_id) = signer.sign_digest_recoverable(digest).unwrap(); + let mut signature = signature.to_vec(); + signature.push(recovery_id.to_byte()); + let trusted = hex::encode(signer.verifying_key().to_sec1_bytes()); + + verify_env_encrypt_public_key(&public_key, &signature, &app_id, timestamp, &trusted, 300) + .unwrap(); + let untrusted = EcdsaSigningKey::random(&mut rand::thread_rng()); + assert!(verify_env_encrypt_public_key( + &public_key, + &signature, + &app_id, + timestamp, + &hex::encode(untrusted.verifying_key().to_sec1_bytes()), + 300, + ) + .is_err()); + } +} diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index 76dceae66..2a456ab12 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -2,26 +2,29 @@ // // SPDX-License-Identifier: Apache-2.0 +use std::sync::Arc; use std::{ collections::{BTreeMap, BTreeSet}, fmt::Display, + io::Write as _, ops::Deref, path::{Path, PathBuf}, - process::Command, + process::{Command, Stdio}, str::FromStr, time::Duration, }; use anyhow::{anyhow, bail, Context, Result}; use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; -use dstack_attest::emit_runtime_event; +use dstack_attest::{default_verifier, emit_runtime_event, set_runtime_event_version}; use dstack_kms_rpc as rpc; use dstack_types::{ + gpu_policy_hash, shared_filenames::{ APP_COMPOSE, APP_KEYS, DECRYPTED_ENV, DECRYPTED_ENV_JSON, ENCRYPTED_ENV, - HOST_SHARED_DIR_NAME, HOST_SHARED_DISK_LABEL, INSTANCE_INFO, SYS_CONFIG, USER_CONFIG, + HOST_SHARED_DIR_NAME, INSTANCE_INFO, SYS_CONFIG, USER_CONFIG, }, - KeyProvider, KeyProviderInfo, + GpuPolicy, KeyProvider, KeyProviderInfo, GPU_ATTESTATION_OUTPUT, }; use fs_err as fs; use luks2::{ @@ -33,13 +36,12 @@ use ra_rpc::{ Attestation, }; use ra_tls::{ - attestation::{AttestationMode, QuoteContentType}, + attestation::{detect_tee_variant, AttestationVerifier, QuoteContentType, TeeVariant}, cert::{generate_ra_cert, CertConfigV2, CertSigningRequestV2, Csr}, }; use rand::Rng as _; -use safe_write::safe_write; +use safe_write::{safe_write, safe_write_with_mode}; use scopeguard::defer; -use semver::{Version, VersionReq}; use serde::{Deserialize, Serialize}; use tracing::{info, warn}; @@ -48,6 +50,7 @@ use crate::{ crypto::dh_decrypt, gen_app_keys_from_seed, host_api::HostApi, + host_shared::{mount_host_shared, unmount_host_shared}, utils::{ deserialize_json_file, sha256, sha256_file, AppCompose, AppKeys, KeyProviderKind, SysConfig, }, @@ -63,6 +66,10 @@ use serde_human_bytes as hex_bytes; use serde_json::Value; use tpm_attest::{self as tpm, TpmContext}; +fn attestation_verifier(sys_config: &SysConfig) -> Result<Arc<AttestationVerifier>> { + Ok(Arc::new(default_verifier(&sys_config.collateral_urls())?)) +} + async fn sign_cert_request( cert_client: &CertRequestClient, key: &KeyPair, @@ -88,12 +95,6 @@ async fn sign_cert_request( mod config_id_verifier; -fn is_unsupported_app_info_quote(err: &anyhow::Error) -> bool { - let message = format!("{err:#}"); - message.contains("Unsupported attestation quote") - || message.contains("unsupported attestation quote for app info decoding") -} - #[derive(clap::Parser)] /// Prepare full disk encryption pub struct SetupArgs { @@ -160,6 +161,7 @@ impl FromStr for FsType { struct DstackOptions { storage_encrypted: bool, storage_fs: FsType, + storage_discard: bool, } fn parse_dstack_options(shared: &HostShared) -> Result<DstackOptions> { @@ -168,6 +170,7 @@ fn parse_dstack_options(shared: &HostShared) -> Result<DstackOptions> { let mut options = DstackOptions { storage_encrypted: true, // Default to encryption enabled storage_fs: FsType::Zfs, // Default to ZFS + storage_discard: true, // Reclaim unused blocks from sparse host images }; for param in cmdline.split_whitespace() { @@ -187,6 +190,7 @@ fn parse_dstack_options(shared: &HostShared) -> Result<DstackOptions> { if let Some(fs) = &shared.app_compose.storage_fs { options.storage_fs = fs.parse().context("Failed to parse storage_fs")?; } + options.storage_discard = shared.app_compose.storage_discard; Ok(options) } @@ -245,42 +249,6 @@ struct HostShared { } impl HostShared { - /// Find block device by volume label - fn find_disk_by_label(label: &str) -> Option<String> { - let label_path = format!("/dev/disk/by-label/{}", label); - if Path::new(&label_path).exists() { - return Some(label_path); - } - - // Fallback: scan /sys/block for devices and check their labels with blkid - if let Ok(entries) = fs::read_dir("/sys/block") { - for entry in entries.flatten() { - let dev_name = entry.file_name(); - let dev_path = format!("/dev/{}", dev_name.to_string_lossy()); - - // Use blkid to check the label - if let Ok(output) = Command::new("blkid") - .arg("-s") - .arg("LABEL") - .arg("-o") - .arg("value") - .arg(&dev_path) - .output() - { - if output.status.success() { - let found_label = - String::from_utf8_lossy(&output.stdout).trim().to_string(); - if found_label == label { - return Some(dev_path); - } - } - } - } - } - - None - } - fn load(host_shared_dir: impl Into<HostShareDir>) -> Result<Self> { let host_shared_dir = host_shared_dir.into(); let sys_config = deserialize_json_file(host_shared_dir.sys_config_file())?; @@ -330,32 +298,8 @@ impl HostShared { std::io::copy(&mut src_io, &mut dst_io)?; Ok(()) }; - cmd! { - info "Mounting host-shared"; - mkdir -p $host_shared_dir; - }?; - - // Try to detect and mount shared disk by label first, fallback to 9p - let disk_device = Self::find_disk_by_label(HOST_SHARED_DISK_LABEL); - let mounted_via_disk = if let Some(dev) = disk_device { - info!("Found shared disk at {}", dev); - let mount_result = cmd! { - info "Attempting to mount shared disk"; - mount -o ro $dev $host_shared_dir; - }; - mount_result.is_ok() - } else { - false - }; - - if !mounted_via_disk { - info!("Shared disk not found, trying 9p virtfs"); - cmd! { - mount -t 9p -o trans=virtio,version=9p2000.L,ro host-shared $host_shared_dir; - }?; - } else { - info!("Successfully mounted shared disk"); - } + info!("Mounting host-shared"); + mount_host_shared(host_shared_dir)?; cmd! { mkdir -p $host_shared_copy_dir; @@ -366,16 +310,14 @@ impl HostShared { copy(INSTANCE_INFO, SZ_1KB * 10, true)?; copy(ENCRYPTED_ENV, SZ_1KB * 256, true)?; copy(USER_CONFIG, SZ_1MB * 50, true)?; - cmd! { - info "Unmounting host-shared"; - umount $host_shared_dir; - }?; + info!("Unmounting host-shared"); + unmount_host_shared(host_shared_dir)?; HostShared::load(host_shared_copy_dir) } } const GATEWAY_CACHE_PATH: &str = "/run/dstack/gateway-cache.json"; -const WG_CONFIG_PATH: &str = "/etc/wireguard/dstack-wg0.conf"; +const GATEWAY_CACHE_PREFIX: &str = "/run/dstack/gateway-cache-"; /// Certificate validity period in seconds (10 days) const CERT_VALIDITY_SECS: u64 = 10 * 24 * 3600; const MAX_SUPPORTED_MANIFEST_VERSION: u32 = 3; @@ -395,27 +337,141 @@ struct GatewayKeyStore { wg_sk: String, /// WireGuard public key wg_pk: String, + /// The gateway URL that last accepted a registration for this cluster. + /// + /// Tried first on the next refresh. Without it the list is walked in + /// configured order every time, which is sticky in the wrong way: every CVM + /// piles onto the first URL, and when that one has an outage the whole + /// fleet moves to the second and then moves *back* the moment the first + /// recovers. Each of those moves rewrites the instance record from a + /// different node's memory, which is exactly what loses per-instance state + /// that only lives there. + /// + /// Scoped to a boot, because the cache is on tmpfs. That is the right + /// scope: a boot regenerates the WireGuard key and re-registers from + /// scratch anyway. + #[serde(default, skip_serializing_if = "Option::is_none")] + last_url: Option<String>, +} + +/// Freshly issued client certificate material for [`GatewayKeyStore::renewed`]. +struct IssuedClientCerts { + client_cert: String, + client_cert_with_quote: String, + client_key: String, + cert_not_after: u64, } impl GatewayKeyStore { - fn load() -> Option<Self> { - let content = fs::read_to_string(GATEWAY_CACHE_PATH).ok()?; + /// The store a certificate renewal produces: fresh certificates plus + /// everything the renewal must carry over from the previous cache — the + /// WireGuard identity, because the gateway maps this peer by its public + /// key, and the registration preference, because the renewed store is + /// saved over the cache before the per-cluster loop reads the preference + /// back, so dropping it here would drop it on disk too and drift the + /// fleet back onto the first configured URL at every renewal. + fn renewed( + cache: Option<Self>, + generate_wg: impl FnOnce() -> Result<(String, String)>, + certs: IssuedClientCerts, + ) -> Result<Self> { + let (wg_sk, wg_pk, last_url) = match cache { + Some(cache) => { + info!("reusing cached WireGuard keys"); + (cache.wg_sk, cache.wg_pk, cache.last_url) + } + None => { + let (wg_sk, wg_pk) = generate_wg()?; + (wg_sk, wg_pk, None) + } + }; + Ok(Self { + client_cert: certs.client_cert, + client_cert_with_quote: certs.client_cert_with_quote, + client_key: certs.client_key, + cert_not_after: certs.cert_not_after, + wg_sk, + wg_pk, + last_url, + }) + } + + fn load_from(path: &Path) -> Option<Self> { + let content = fs::read_to_string(path).ok()?; serde_json::from_str(&content).ok() } - fn save(&self) -> Result<()> { + fn load_from_default() -> Option<Self> { + Self::load_from(Path::new(GATEWAY_CACHE_PATH)) + } + + fn save_to(&self, path: &Path) -> Result<()> { let content = serde_json::to_string(self).context("Failed to serialize gateway cache")?; - safe_write(GATEWAY_CACHE_PATH, &content).context("Failed to write gateway cache")?; + safe_write_with_mode(path, &content, 0o600).context("Failed to write gateway cache")?; Ok(()) } + fn save_to_default(&self) -> Result<()> { + self.save_to(Path::new(GATEWAY_CACHE_PATH)) + } + + fn is_cert_valid_at(&self, now: u64) -> bool { + // Valid if at least 10 minutes remaining. + now.saturating_add(600) < self.cert_not_after + } + fn is_cert_valid(&self) -> bool { let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0); - // Valid if at least 10 minutes remaining - now + 600 < self.cert_not_after + self.is_cert_valid_at(now) + } +} + +fn gateway_rpc_url(base: &str) -> String { + let base = base.trim_end_matches('/'); + if base.ends_with("/prpc") { + base.to_string() + } else { + format!("{base}/prpc") + } +} + +#[derive(Debug)] +struct GatewayTarget { + name: String, + urls: Vec<String>, +} + +struct PreparedGatewayCluster { + name: String, + index: usize, + key_store: GatewayKeyStore, + response: RegisterCvmResponse, +} + +fn wireguard_endpoint_hosts(config: &str) -> Result<Vec<String>> { + config + .lines() + .filter_map(|line| line.trim().strip_prefix("Endpoint = ")) + .map(|endpoint| { + endpoint + .rsplit_once(':') + .map(|(host, _)| host.trim_matches(['[', ']']).to_string()) + .context("invalid WireGuard endpoint") + }) + .collect() +} + +fn remove_partial_wireguard_config(path: &str, cluster: &str) { + if let Err(error) = fs::remove_file(path) { + if error.kind() != std::io::ErrorKind::NotFound { + warn!( + cluster = cluster, + "failed to remove partially applied WireGuard config: {error}" + ); + } } } @@ -435,20 +491,20 @@ impl<'a> GatewayContext<'a> { gateway_url: &str, client_key: &str, client_cert: &str, + gateway_app_id: &str, ) -> Result<GatewayClient<RaClient>> { - let url = format!("{}/prpc", gateway_url); + let url = gateway_rpc_url(gateway_url); let ca_cert = self.keys.ca_cert.clone(); let cert_validator = AppIdValidator { - allowed_app_id: self.keys.gateway_app_id.clone(), + allowed_app_id: gateway_app_id.to_string(), }; let client = RaClientConfig::builder() .remote_uri(url) - .maybe_pccs_url(self.shared.sys_config.pccs_url.clone()) .tls_client_cert(client_cert.to_string()) .tls_client_key(client_key.to_string()) .tls_ca_cert(ca_cert) .tls_built_in_root_certs(false) - .tls_no_check(self.keys.gateway_app_id == "any") + .tls_no_check(gateway_app_id == "any") .verify_server_attestation(false) .cert_validator(Box::new(move |cert| cert_validator.validate(cert))) .build() @@ -461,6 +517,7 @@ impl<'a> GatewayContext<'a> { &self, gateway_url: &str, key_store: &GatewayKeyStore, + gateway_app_id: &str, ) -> Result<RegisterCvmResponse> { let port_policy = RpcPortPolicy { ports: self @@ -476,12 +533,18 @@ impl<'a> GatewayContext<'a> { .collect(), restrict_mode: self.shared.app_compose.port_policy.restrict_mode, }; - let client = - self.create_gateway_client(gateway_url, &key_store.client_key, &key_store.client_cert)?; + let health_check = health_check_requested(&self.shared.app_compose); + let client = self.create_gateway_client( + gateway_url, + &key_store.client_key, + &key_store.client_cert, + gateway_app_id, + )?; let result = client .register_cvm(RegisterCvmRequest { client_public_key: key_store.wg_pk.clone(), port_policy: Some(port_policy.clone()), + health_check, }) .await .context("Failed to register CVM"); @@ -498,11 +561,13 @@ impl<'a> GatewayContext<'a> { gateway_url, &key_store.client_key, &key_store.client_cert_with_quote, + gateway_app_id, )?; client .register_cvm(RegisterCvmRequest { client_public_key: key_store.wg_pk.clone(), port_policy: Some(port_policy), + health_check, }) .await .context("Failed to register CVM") @@ -510,7 +575,7 @@ impl<'a> GatewayContext<'a> { async fn get_or_generate_key_store(&self) -> Result<GatewayKeyStore> { // Try to load existing cache - let cache = GatewayKeyStore::load(); + let cache = GatewayKeyStore::load_from_default(); // If cache is fully valid, return it if let Some(ref cache) = cache { @@ -520,18 +585,6 @@ impl<'a> GatewayContext<'a> { } } - // Reuse WireGuard keys from cache if available, otherwise generate new ones - let (wg_sk, wg_pk) = if let Some(ref cache) = cache { - info!("Reusing cached WireGuard keys"); - (cache.wg_sk.clone(), cache.wg_pk.clone()) - } else { - info!("Generating new WireGuard keys"); - let sk = cmd!(wg genkey)?; - let pk = - cmd!(echo $sk | wg pubkey).or(Err(anyhow!("Failed to generate public key")))?; - (sk, pk) - }; - // Request new client certificates info!("Requesting new client certificates"); let now = std::time::SystemTime::now() @@ -539,9 +592,10 @@ impl<'a> GatewayContext<'a> { .map(|d| d.as_secs()) .unwrap_or(0); let cert_not_after = now + CERT_VALIDITY_SECS; + let verifier = attestation_verifier(&self.shared.sys_config)?; let cert_client = CertRequestClient::create( self.keys, - self.shared.sys_config.pccs_url.as_deref(), + verifier, self.shared.sys_config.vm_config.clone(), ) .await @@ -585,14 +639,49 @@ impl<'a> GatewayContext<'a> { .context("Failed to request cert with quote")?; let client_cert_with_quote = certs_with_quote.join("\n"); - Ok(GatewayKeyStore { - client_cert, - client_cert_with_quote, - client_key, - cert_not_after, - wg_sk, - wg_pk, - }) + GatewayKeyStore::renewed( + cache, + || { + info!("generating new WireGuard keys"); + let sk = cmd!(wg genkey)?; + let pk = + cmd!(echo $sk | wg pubkey).or(Err(anyhow!("Failed to generate public key")))?; + Ok((sk, pk)) + }, + IssuedClientCerts { + client_cert, + client_cert_with_quote, + client_key, + cert_not_after, + }, + ) + } + + fn key_store_for_additional_cluster( + &self, + name: &str, + certificate_source: &GatewayKeyStore, + ) -> Result<(GatewayKeyStore, PathBuf)> { + let path = PathBuf::from(format!("{GATEWAY_CACHE_PREFIX}{name}.json")); + if let Some(cache) = GatewayKeyStore::load_from(&path) { + if cache.is_cert_valid() { + info!(cluster = name, "Using cached gateway cluster key store"); + return Ok((cache, path)); + } + } + let old = GatewayKeyStore::load_from(&path); + let (wg_sk, wg_pk) = if let Some(old) = old { + (old.wg_sk, old.wg_pk) + } else { + let sk = cmd!(wg genkey)?; + let pk = + cmd!(echo $sk | wg pubkey).or(Err(anyhow!("Failed to generate public key")))?; + (sk, pk) + }; + let mut key_store = certificate_source.clone(); + key_store.wg_sk = wg_sk; + key_store.wg_pk = wg_pk; + Ok((key_store, path)) } async fn setup(&self, force: bool) -> Result<()> { @@ -604,110 +693,226 @@ impl<'a> GatewayContext<'a> { bail!("Missing allowed dstack-gateway app id"); } - info!("Setting up dstack-gateway"); - - // Get or generate key store (includes WireGuard keys and client certificate) - let key_store = self.get_or_generate_key_store().await?; + let targets = self.gateway_targets()?; + let uses_explicit_clusters = !self.shared.sys_config.gateway_clusters.is_empty(); + info!(clusters = targets.len(), "Setting up dstack-gateway"); - if self.shared.sys_config.gateway_urls.is_empty() { - bail!("Missing gateway urls"); + // Certificates are valid for every gateway identity authorized by KMS, + // while each cluster receives a distinct WireGuard identity. + let primary_key_store = self.get_or_generate_key_store().await?; + if let Err(err) = primary_key_store.save_to_default() { + warn!("failed to save gateway cache: {err:?}"); } - // Read config and make API call - let response = 'out: { - let mut error = anyhow!("unknown error"); - for (i, url) in self.shared.sys_config.gateway_urls.iter().enumerate() { - let response = self.register_cvm(url, &key_store).await; - match response { - Ok(response) => { - break 'out response; + + let mut errors = Vec::new(); + for (index, target) in targets.iter().enumerate() { + let key_store_result = if index == 0 && !uses_explicit_clusters { + Ok((primary_key_store.clone(), PathBuf::from(GATEWAY_CACHE_PATH))) + } else { + self.key_store_for_additional_cluster(&target.name, &primary_key_store) + }; + let (mut key_store, cache_path) = match key_store_result { + Ok(value) => value, + Err(err) => { + errors.push(format!("{}: {err:#}", target.name)); + continue; + } + }; + carry_cluster_preference(&mut key_store, &cache_path, &target.urls); + if let Err(err) = key_store.save_to(&cache_path) { + warn!(cluster = %target.name, "failed to save gateway cluster cache: {err:?}"); + } + + let mut first_error = None; + let mut response = None; + let mut accepted_by = None; + for url in order_by_stickiness(&target.urls, key_store.last_url.as_deref()) { + match self + .register_cvm(url, &key_store, &self.keys.gateway_app_id) + .await + { + Ok(value) => { + response = Some(value); + accepted_by = Some(url.clone()); + break; } Err(err) => { - warn!("Failed to register CVM: {err:?}, retrying with next dstack-gateway"); - if i == 0 { - error = err; + warn!(cluster = %target.name, %url, "Failed to register CVM: {err:?}"); + if first_error.is_none() { + first_error = Some(err); } } } } - return Err(error).context("Failed to register CVM, all dstack-gateway urls are down"); - }; - let mut wg_info = response.wg.context("Missing wg info")?; + record_accepting_url(&target.name, &mut key_store, &cache_path, accepted_by); + let Some(response) = response else { + errors.push(format!( + "{}: {:#}", + target.name, + first_error.unwrap_or_else(|| anyhow!("no gateway URLs configured")) + )); + continue; + }; + let cluster = PreparedGatewayCluster { + name: target.name.clone(), + index, + key_store, + response, + }; + if let Err(err) = self.apply_wireguard(cluster, force) { + errors.push(format!("{}: {err:#}", target.name)); + } + } + if errors.is_empty() { + Ok(()) + } else { + bail!("failed to refresh gateway clusters: {}", errors.join("; ")) + } + } - let client_ip = &wg_info.client_ip; + fn gateway_targets(&self) -> Result<Vec<GatewayTarget>> { + if !self.shared.sys_config.gateway_urls.is_empty() + && !self.shared.sys_config.gateway_clusters.is_empty() + { + warn!("both gateway_urls and gateway_clusters are configured; ignoring gateway_urls"); + } + let targets = if self.shared.sys_config.gateway_clusters.is_empty() { + vec![GatewayTarget { + name: "default".to_string(), + urls: self.shared.sys_config.gateway_urls.clone(), + }] + } else { + self.shared + .sys_config + .gateway_clusters + .iter() + .map(|cluster| GatewayTarget { + name: cluster.name.clone(), + urls: cluster.urls.clone(), + }) + .collect() + }; + let mut names = std::collections::HashSet::new(); + for target in &targets { + if target.name.is_empty() + || !target + .name + .bytes() + .all(|c| c.is_ascii_alphanumeric() || c == b'-' || c == b'_') + { + bail!("invalid gateway cluster name: {}", target.name); + } + if !names.insert(target.name.as_str()) { + bail!("duplicate gateway cluster name: {}", target.name); + } + if target.urls.is_empty() { + bail!("gateway cluster {} has no URLs", target.name); + } + } + Ok(targets) + } - // Sort peers by public key for consistent config generation + fn apply_wireguard(&self, mut cluster: PreparedGatewayCluster, force: bool) -> Result<()> { + let interface = format!("dstack-wg{}", cluster.index); + let config_path = format!("/etc/wireguard/{interface}.conf"); + let listen_port = 9182_u16 + .checked_add( + cluster + .index + .try_into() + .context("too many gateway clusters")?, + ) + .context("too many gateway clusters")?; + let mut wg_info = cluster.response.wg.take().context("missing wg info")?; wg_info.servers.sort_by(|a, b| a.pk.cmp(&b.pk)); - - // Create WireGuard config - let wg_listen_port = "9182"; let mut new_config = format!( - "[Interface]\n\ - PrivateKey = {}\n\ - ListenPort = {wg_listen_port}\n\ - Address = {client_ip}/32\n\n", - key_store.wg_sk + "[Interface]\nPrivateKey = {}\nListenPort = {listen_port}\nAddress = {}/32\n\n", + cluster.key_store.wg_sk, wg_info.client_ip ); for WireGuardPeer { pk, ip, endpoint } in &wg_info.servers { let ip = ip.split('/').next().unwrap_or_default(); new_config.push_str(&format!( - "[Peer]\n\ - PublicKey = {pk}\n\ - AllowedIPs = {ip}/32\n\ - Endpoint = {endpoint}\n\ - PersistentKeepalive = 25\n", + "[Peer]\nPublicKey = {pk}\nAllowedIPs = {ip}/32\nEndpoint = {endpoint}\nPersistentKeepalive = 25\n" )); } - - // Save cache - if let Err(e) = key_store.save() { - warn!("Failed to save gateway cache: {e:?}"); + let old_config = fs::read_to_string(&config_path).ok(); + if !force && old_config.as_ref() == Some(&new_config) { + info!(cluster = %cluster.name, "WireGuard config unchanged"); + return Ok(()); } - - // Check if config has changed (skip check if force is set) - if !force { - let current_config = fs::read_to_string(WG_CONFIG_PATH).ok(); - if current_config.as_ref() == Some(&new_config) { - info!("WireGuard config unchanged, skipping reconfiguration"); - return Ok(()); + let new_endpoints = wireguard_endpoint_hosts(&new_config)?; + let applied = self.apply_wireguard_config( + &interface, + &config_path, + listen_port, + &new_config, + &new_endpoints, + ); + if let Err(error) = applied { + // Registration and refresh are independent per cluster. Preserve + // this cluster's last-known-good config if applying its replacement + // fails; a cluster with no prior config removes the false marker so + // the checker takes its fast missing-config retry path. + if let Some(old_config) = old_config { + match wireguard_endpoint_hosts(&old_config).and_then(|endpoints| { + self.apply_wireguard_config( + &interface, + &config_path, + listen_port, + &old_config, + &endpoints, + ) + }) { + Ok(()) => { + warn!(cluster = %cluster.name, "restored previous WireGuard config after refresh failure") + } + Err(rollback_error) => { + warn!(cluster = %cluster.name, "failed to restore previous WireGuard config: {rollback_error:#}"); + remove_partial_wireguard_config(&config_path, &cluster.name); + } + } + } else { + remove_partial_wireguard_config(&config_path, &cluster.name); } + return Err(error); } + Ok(()) + } - let wg_dir = Path::new("/etc/wireguard"); - fs::create_dir_all(wg_dir)?; - fs::write(wg_dir.join("dstack-wg0.conf"), &new_config)?; - - cmd! { - chmod 600 $wg_dir/dstack-wg0.conf; - ignore wg-quick down dstack-wg0; - }?; - - // Setup WireGuard iptables rules - cmd! { - // Create the chain if it doesn't exist - ignore iptables -N DSTACK_WG 2>/dev/null; - // Flush the chain - iptables -F DSTACK_WG; - // Remove any existing jump rule - ignore iptables -D INPUT -p udp --dport $wg_listen_port -j DSTACK_WG 2>/dev/null; - // Insert the new jump rule at the beginning of the INPUT chain - iptables -I INPUT -p udp --dport $wg_listen_port -j DSTACK_WG - }?; - - for peer in &wg_info.servers { - // Avoid issues with field-access in the macro by binding the IP to a local variable. - let endpoint_ip = peer - .endpoint - .split(':') - .next() - .context("Invalid wireguard endpoint")?; - cmd!(iptables -A DSTACK_WG -s $endpoint_ip -j ACCEPT)?; + fn apply_wireguard_config( + &self, + interface: &str, + config_path: &str, + listen_port: u16, + config: &str, + endpoint_hosts: &[String], + ) -> Result<()> { + safe_write_with_mode(config_path, config, 0o600) + .context("failed to write WireGuard config")?; + cmd!(ignore wg-quick down $interface)?; + + // Docker also updates iptables throughout boot. Bound every lock wait + // so a transient xtables.lock holder neither breaks the cluster update + // nor stalls the checker indefinitely. + let xtables_wait = "5"; + let chain = format!("DSTACK_WG{}", interface.trim_start_matches("dstack-wg")); + if interface == "dstack-wg0" { + // Remove the pre-multi-cluster chain after upgrading. The new + // per-interface chain below owns the same listen port. + cmd!(ignore iptables -w $xtables_wait -D INPUT -p udp --dport $listen_port -j DSTACK_WG 2>/dev/null)?; + cmd!(ignore iptables -w $xtables_wait -F DSTACK_WG 2>/dev/null)?; + cmd!(ignore iptables -w $xtables_wait -X DSTACK_WG 2>/dev/null)?; } - - // Drop any UDP packets that don't come from an allowed IP. - cmd!(iptables -A DSTACK_WG -j DROP)?; - - info!("Starting WireGuard"); - cmd!(wg-quick up dstack-wg0)?; + cmd!(ignore iptables -w $xtables_wait -N $chain 2>/dev/null)?; + cmd!(iptables -w $xtables_wait -F $chain)?; + cmd!(ignore iptables -w $xtables_wait -D INPUT -p udp --dport $listen_port -j $chain 2>/dev/null)?; + cmd!(iptables -w $xtables_wait -I INPUT -p udp --dport $listen_port -j $chain)?; + for endpoint_host in endpoint_hosts { + cmd!(iptables -w $xtables_wait -A $chain -s $endpoint_host -j ACCEPT)?; + } + cmd!(iptables -w $xtables_wait -A $chain -j DROP)?; + info!(%interface, "starting WireGuard"); + cmd!(wg-quick up $interface)?; Ok(()) } } @@ -799,22 +1004,17 @@ fn verify_app_compose_policy(shared: &HostShared) -> Result<()> { let Some(requirements) = app_compose.requirements.as_ref() else { return Ok(()); }; - if requirements.os_version.is_some() { - let current_os_version = - read_current_os_version().context("Failed to read current dstack OS version")?; - verify_os_version_requirement(app_compose, ¤t_os_version)?; - } if let Some(platforms) = requirements.platforms.as_deref() { if platforms.is_empty() { bail!("Unsupported attestation platform: requirements.platforms is empty"); } let current_platform = - AttestationMode::detect().context("failed to detect current attestation platform")?; + detect_tee_variant().context("failed to detect current attestation platform")?; verify_platform_requirements(app_compose, current_platform)?; } if requirements.tdx_measure_acpi_tables.is_some() { let current_platform = - AttestationMode::detect().context("failed to detect current attestation platform")?; + detect_tee_variant().context("failed to detect current attestation platform")?; verify_tdx_measure_acpi_tables_requirement( app_compose, &sys_config.vm_config, @@ -832,6 +1032,93 @@ fn verify_app_compose_policy(shared: &HostShared) -> Result<()> { Ok(()) } +/// The order to try a cluster's gateway URLs in, last accepting one first. +/// +/// Walking the configured list every time is sticky in the wrong way. Every CVM +/// piles onto the first URL, so one node takes all registration traffic; and +/// when that node has an outage the whole fleet moves to the second URL and +/// then moves *back* the instant the first recovers. Every one of those moves +/// rewrites the instance record from a different node's memory, which is how +/// per-instance state that lives only there gets lost. +/// +/// Preferring the last node that accepted turns both into one-time events: a +/// CVM that moved stays moved, and the fleet spreads over the outage rather +/// than snapping back together. +/// +/// `last` is only a preference. Everything else still follows in configured +/// order, so a URL that is down costs one failed attempt and nothing more. +fn order_by_stickiness<'a>(urls: &'a [String], last: Option<&str>) -> Vec<&'a String> { + let mut ordered = Vec::with_capacity(urls.len()); + if let Some(last) = last { + ordered.extend(urls.iter().filter(|url| url.as_str() == last)); + } + ordered.extend(urls.iter().filter(|url| Some(url.as_str()) != last)); + ordered +} + +/// Carry the cluster's own remembered URL into a freshly resolved key store. +/// +/// The preference is per cluster. The store handed in may have just been +/// rebuilt by a certificate renewal, or cloned from the primary's for an +/// additional cluster — either way the preference that counts is the one this +/// cluster's cache holds, and only while the operator still lists the URL: a +/// URL removed from the config is dropped rather than resurrected. +fn carry_cluster_preference( + key_store: &mut GatewayKeyStore, + cache_path: &Path, + configured: &[String], +) { + key_store.last_url = GatewayKeyStore::load_from(cache_path) + .and_then(|cached| cached.last_url) + .filter(|url| configured.iter().any(|configured| configured == url)); +} + +/// Persist the URL that accepted this round's registration, if it changed. +/// +/// Only an acceptance updates the preference. A round where every URL failed +/// says nothing about where the instance record lives, and clearing on it +/// would have one bad tick — a full-cluster outage, or a blip in this CVM's +/// own networking — erase the fleet's spread and pile everyone back onto the +/// first configured URL at recovery. +fn record_accepting_url( + cluster: &str, + key_store: &mut GatewayKeyStore, + cache_path: &Path, + accepted_by: Option<String>, +) { + let Some(url) = accepted_by else { + return; + }; + if key_store.last_url.as_deref() == Some(url.as_str()) { + return; + } + if key_store.last_url.is_some() { + info!(cluster, %url, "gateway registration moved"); + } + key_store.last_url = Some(url); + if let Err(err) = key_store.save_to(cache_path) { + warn!(cluster, "failed to record the accepting gateway: {err:?}"); + } +} + +/// Whether this app asked the gateway to gate its traffic on health. +/// +/// Opt-in, not a build-time fact: the guest agent only refreshes a verdict when +/// the app asked for one, so telling the gateway to poll an app that did not +/// would cost every gateway node a round trip per interval to be told what it +/// already assumes. +/// +/// This is the only hop between the app's manifest and the gateway's behaviour, +/// which is why it is a named function rather than an expression inside the +/// registration call: getting it wrong makes the whole feature inert fleet-wide +/// with nothing to show for it. +fn health_check_requested(app_compose: &AppCompose) -> bool { + app_compose + .requirements + .as_ref() + .is_some_and(|requirements| requirements.health_check) +} + fn verify_manifest_feature_requirements(app_compose: &AppCompose) -> Result<()> { let manifest_version = verify_manifest_version(app_compose)?; if app_compose.requirements.is_some() && manifest_version < MANIFEST_VERSION_3 { @@ -839,35 +1126,78 @@ fn verify_manifest_feature_requirements(app_compose: &AppCompose) -> Result<()> "requirements requires manifest_version >= {MANIFEST_VERSION_3}; use string manifest_version \"{MANIFEST_VERSION_3}\" so older guests fail closed" ); } + if app_compose.runner == "nerdctl-compose" && manifest_version < MANIFEST_VERSION_3 { + bail!( + "nerdctl-compose requires manifest_version >= {MANIFEST_VERSION_3}; use string manifest_version \"{MANIFEST_VERSION_3}\" so older guests fail closed" + ); + } + if app_compose.init_script.len() > 1 && manifest_version < MANIFEST_VERSION_3 { + bail!( + "multiple init scripts require manifest_version >= {MANIFEST_VERSION_3}; use string manifest_version \"{MANIFEST_VERSION_3}\" so older guests fail closed" + ); + } + if app_compose.runner != "nerdctl-compose" && app_compose.snapshotter.is_some() { + bail!("snapshotter is only supported by the nerdctl-compose runner"); + } + verify_health_check_requirement(app_compose)?; Ok(()) } -fn verify_os_version_requirement(app_compose: &AppCompose, current_os_version: &str) -> Result<()> { +/// Reject a health-gating request this guest could not act on. +/// +/// Only checks that need nothing but the manifest. The obvious third +/// check -- "does any service actually declare a `healthcheck:`?" -- is +/// deliberately *not* here, even though an app whose containers declare none +/// reports healthy forever, which is the failure this is meant to remove. +/// +/// It is not here because at this point the only thing available is the raw +/// compose text, and reading it is not the same question as the one the runtime +/// answers. Scanning YAML for the key misses a `HEALTHCHECK` in the Dockerfile, +/// misses a `<<:` merge from a shared anchor (the loader does not expand merge +/// keys, so the service reads as having no `healthcheck` at all), misses +/// `extends:` and multi-file overrides -- and *passes* `healthcheck: {disable: +/// true}` and `test: ["NONE"]`, which are exactly the configurations that +/// produce no verdict at runtime. It rejects working deployments and admits +/// broken ones. +/// +/// The guest agent asks the runtime instead, which is the authority, and +/// reports "nothing here can be judged" as unhealthy-with-a-reason. That +/// surfaces in the gateway log and on the dashboard rather than as a boot +/// failure, and it is right in all of the cases above. +fn verify_health_check_requirement(app_compose: &AppCompose) -> Result<()> { let Some(requirements) = app_compose.requirements.as_ref() else { return Ok(()); }; - let Some(os_version) = requirements.os_version.as_deref() else { + if !requirements.health_check { + // A path without gating enabled is inert, not an error: an operator + // turning gating off during an incident should not have to also delete + // the path they will want back. return Ok(()); - }; - let os_version_req = VersionReq::parse(os_version) - .with_context(|| format!("Invalid requirements.os_version: {os_version}"))?; - let current_os_version = Version::parse(current_os_version) - .with_context(|| format!("Invalid current dstack OS version: {current_os_version}"))?; - if !os_version_req.matches(¤t_os_version) { - bail!( - "Unsupported dstack OS version: current {current_os_version}, required {os_version_req}" - ); } - info!( - "dstack OS version requirement satisfied: current={}, requirement={}", - current_os_version, os_version_req - ); + match requirements.health_status_file.as_deref() { + Some(path) => { + if !path.starts_with('/') { + bail!("requirements.health_status_file must be an absolute path: {path}"); + } + } + None => { + // The container fallback has nothing to look at for a runner that + // starts no containers, so it would answer "healthy" unconditionally. + if app_compose.runner != "docker-compose" && app_compose.runner != "nerdctl-compose" { + bail!( + "requirements.health_check needs health_status_file for the {} runner; \ + without containers to inspect there is nothing to judge", + app_compose.runner + ); + } + } + } Ok(()) } fn verify_platform_requirements( app_compose: &AppCompose, - current_platform: AttestationMode, + current_platform: TeeVariant, ) -> Result<()> { let Some(requirements) = app_compose.requirements.as_ref() else { return Ok(()); @@ -895,7 +1225,7 @@ fn verify_platform_requirements( ); } -fn parse_requirement_platform(platform: &str, index: usize) -> Result<AttestationMode> { +fn parse_requirement_platform(platform: &str, index: usize) -> Result<TeeVariant> { serde_json::from_value(serde_json::Value::String(platform.to_string())) .with_context(|| format!("Invalid requirements.platforms[{index}]: {platform}")) } @@ -907,7 +1237,7 @@ fn format_requirement_platforms(platforms: &[String]) -> String { fn verify_tdx_measure_acpi_tables_requirement( app_compose: &AppCompose, vm_config: &str, - current_platform: AttestationMode, + current_platform: TeeVariant, ) -> Result<()> { let Some(measure_acpi_tables) = app_compose .requirements @@ -916,7 +1246,7 @@ fn verify_tdx_measure_acpi_tables_requirement( else { return Ok(()); }; - if current_platform != AttestationMode::DstackTdx { + if current_platform != TeeVariant::DstackTdx { return Ok(()); } let vm_config: dstack_types::VmConfig = serde_json::from_str(vm_config) @@ -988,60 +1318,10 @@ fn launch_token_from_user_config(user_config: &str) -> Result<String> { Ok(token.to_string()) } -fn read_current_os_version() -> Result<String> { - const OS_RELEASE_PATHS: &[&str] = &["/etc/os-release", "/usr/lib/os-release"]; - for path in OS_RELEASE_PATHS { - let content = match fs::read_to_string(path) { - Ok(content) => content, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, - Err(err) => return Err(err).with_context(|| format!("Failed to read {path}")), - }; - if let Some(version) = os_release_value(&content, "VERSION_ID") { - return Ok(version); - } - } - bail!("VERSION_ID not found in /etc/os-release or /usr/lib/os-release") -} - -fn os_release_value(content: &str, key: &str) -> Option<String> { - for line in content.lines() { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - let Some((k, v)) = line.split_once('=') else { - continue; - }; - if k == key { - return Some(unquote_os_release_value(v)); - } - } - None -} - -fn unquote_os_release_value(value: &str) -> String { - let value = value.trim(); - if let Some(inner) = value.strip_prefix('"').and_then(|v| v.strip_suffix('"')) { - // Double-quoted: a backslash escapes the next character. - let mut unescaped = String::with_capacity(inner.len()); - let mut chars = inner.chars(); - while let Some(c) = chars.next() { - match c { - '\\' => unescaped.push(chars.next().unwrap_or('\\')), - _ => unescaped.push(c), - } - } - return unescaped; - } - if let Some(inner) = value.strip_prefix('\'').and_then(|v| v.strip_suffix('\'')) { - // Single-quoted: shell single quotes have no escape sequences. - return inner.to_string(); - } - value.to_string() -} - pub async fn cmd_sys_setup(args: SetupArgs) -> Result<()> { let stage0 = Stage0::load(&args)?; + set_runtime_event_version(stage0.shared.app_compose.event_log_version) + .context("failed to configure runtime event version")?; let vmm = stage0.host_api(); let result = do_sys_setup(stage0).await; if let Err(err) = &result { @@ -1061,93 +1341,278 @@ async fn do_sys_setup(stage0: Stage0<'_>) -> Result<()> { } else { info!("System time will be synchronized by chronyd in background"); } - stage0 - .setup_gpu() - .await - .context("Failed to verify GPU TEE attestation")?; let stage1 = stage0.setup_fs().await?; stage1.setup().await } -/// GPU TEE attestation gate (`requirements.attest_gpu`, defaults to true). +/// GPU TEE attestation gate (`requirements.gpu_policy.attest_gpu`, defaults to +/// true). /// /// Runs before key provisioning so a CVM whose GPU cannot prove it is a -/// genuine, CC-enabled NVIDIA TEE never gets its app keys. The GPU -/// "ready" state (`nvidia-smi conf-compute -srs 1`) is only set from here — -/// nvidia-persistenced deliberately does not set it — so CUDA work cannot be -/// submitted to an unverified GPU either. +/// genuine, CC-enabled NVIDIA TEE never gets its app keys. An optional +/// application policy is measured and evaluated after `compose-hash`. The GPU +/// "ready" state is only set through NVML from here — nvidia-persistenced +/// deliberately does not set it — so CUDA work cannot be submitted to an +/// unverified GPU either. mod gpu { use super::*; - const NVATTEST: &str = "/usr/bin/nvattest"; - const NVIDIA_SMI: &str = "/usr/bin/nvidia-smi"; - const ATTESTATION_OUTPUT: &str = "/run/nvidia-gpu-attestation/attestation.out"; - const ATTESTATION_TIMEOUT: Duration = Duration::from_secs(300); - const NVIDIA_SMI_TIMEOUT: Duration = Duration::from_secs(60); - - /// True if a passed-through NVIDIA GPU is present, detected via sysfs PCI - /// (vendor 0x10de, class VGA 0x0300xx or 3D controller 0x0302xx) so it - /// works even before the nvidia driver is loaded. Fails (rather than - /// reporting "no GPU") when the PCI bus cannot be enumerated, so a broken - /// /sys cannot bypass the attestation gate. - pub(super) fn nvidia_gpu_present() -> Result<bool> { - let entries = - fs::read_dir("/sys/bus/pci/devices").context("failed to enumerate PCI devices")?; - Ok(entries.filter_map(|e| e.ok()).any(|dev| { - let read = |name: &str| { - fs::read_to_string(dev.path().join(name)) - .unwrap_or_default() - .trim() - .to_string() - }; - read("vendor") == "0x10de" - && matches!(read("class").get(..6), Some("0x0300") | Some("0x0302")) - })) + const EVENT_VERSION: u32 = 2; + const POLICY_ENTRYPOINT: &str = "data.policy.nv_match"; + /// Bound Rego evaluation so a runaway application policy cannot hang boot. + const POLICY_TIMEOUT: Duration = Duration::from_secs(10); + + use lspci::sysfs::GpuInventory; + + #[derive(Debug, Serialize)] + struct GpuAttestationEvent { + version: u32, + provider: &'static str, + devices: u32, + cc_mode: &'static str, + devtools: bool, + evidence_sha256: String, } - /// Run a GPU tool with a bounded timeout so a wedged driver/GPU cannot - /// hang the boot indefinitely (dstack-prepare is a oneshot unit with no - /// start timeout of its own). - async fn run_command( - program: &str, - args: &[&str], - timeout: Duration, - ) -> Result<std::process::Output> { - tokio::time::timeout( - timeout, - tokio::process::Command::new(program).args(args).output(), - ) - .await - .with_context(|| format!("{program} timed out"))? - .with_context(|| format!("failed to run {program}")) + pub(super) struct GpuAttestationResult { + claims: Vec<Value>, + parsed_claims: Vec<NvidiaGpuClaim>, + output: Vec<u8>, + devices: u32, } - /// Mark the GPU as ready to accept work. Only meaningful (and only - /// succeeds) when the GPU runs in CC mode. - pub(super) async fn set_gpu_ready_state() -> Result<()> { - let output = run_command( - NVIDIA_SMI, - &["conf-compute", "-srs", "1"], - NVIDIA_SMI_TIMEOUT, - ) - .await?; - if !output.status.success() { + impl GpuAttestationResult { + pub(super) fn claims(&self) -> &[Value] { + &self.claims + } + + pub(super) fn event(&self, devtools: bool) -> Result<Vec<u8>> { + attestation_event(&self.output, self.devices, devtools) + } + + pub(super) fn verify_claim_policy( + &self, + state: &GpuState, + policy: &GpuPolicy, + ) -> Result<()> { + verify_claim_policy(&self.parsed_claims, &state.devices, policy) + } + } + + pub(super) struct GpuState { + nvml: nvml_wrapper::Nvml, + devices: Vec<GpuDeviceState>, + } + + impl GpuState { + pub(super) fn any_devtools(&self) -> bool { + self.devices.iter().any(|device| device.devtools) + } + + pub(super) fn set_ready(&self) -> Result<()> { + set_gpu_ready_state_with_nvml(&self.nvml) + } + } + + #[derive(Debug, Clone, Copy)] + struct GpuDeviceState { + cc_enabled: bool, + devtools: bool, + } + + #[derive(Deserialize)] + struct NvattestOutput { + result_code: i64, + claims: Vec<Value>, + } + + #[derive(Debug, Deserialize)] + struct NvidiaGpuClaim { + #[serde(rename = "x-nvidia-device-type")] + device_type: String, + eat_nonce: String, + #[serde(rename = "x-nvidia-gpu-attestation-report-nonce-match")] + nonce_match: bool, + measres: String, + secboot: bool, + dbgstat: NvidiaGpuDebugStatus, + } + + #[derive(Debug, Deserialize, PartialEq, Eq)] + #[serde(rename_all = "lowercase")] + enum NvidiaGpuDebugStatus { + Disabled, + Enabled, + } + + struct ValidatedClaims { + raw: Vec<Value>, + parsed: Vec<NvidiaGpuClaim>, + } + + /// Count passed-through display-class GPUs through sysfs so devices which + /// the NVIDIA driver did not bind cannot be hidden from the gate. Reading + /// the inventory is fail-closed: a mixed NVIDIA/non-NVIDIA set must not be + /// represented by an attestation result for only the NVIDIA subset. + /// + /// The scan itself lives in `lspci::sysfs` because the guest agent's GPU + /// telemetry gate needs the same answer with a different failure policy. + pub(super) fn gpu_inventory() -> Result<GpuInventory> { + lspci::sysfs::gpu_inventory() + } + + pub(super) fn nvidia_gpu_count(inventory: GpuInventory) -> Result<u32> { + if inventory.total != inventory.nvidia { bail!( - "nvidia-smi conf-compute -srs 1 failed ({}): {}", - output.status, - truncated_lossy(&output.stderr, 512), + "unsupported non-NVIDIA GPU attached: found {} display GPUs, {} NVIDIA", + inventory.total, + inventory.nvidia ); } + Ok(inventory.nvidia) + } + + fn init_nvml(expected_devices: u32) -> Result<nvml_wrapper::Nvml> { + let nvml = nvml_wrapper::Nvml::init().context("failed to initialize NVML")?; + let devices = nvml + .device_count() + .context("failed to get NVML GPU count")?; + if devices != expected_devices { + bail!("nvml GPU count mismatch: expected {expected_devices}, got {devices}"); + } + Ok(nvml) + } + + fn set_gpu_ready_state_with_nvml(nvml: &nvml_wrapper::Nvml) -> Result<()> { + // nvml-wrapper exposes nvmlSystemSetConfComputeGpusReadyState through + // Device, but the transition applies to all CC GPUs in the system. + let first = nvml + .device_by_index(0) + .context("failed to get first NVML GPU")?; + first + .set_confidential_compute_state(true) + .context("failed to set GPU ready state")?; info!("GPU ready state set"); Ok(()) } - /// Run local GPU attestation via nvattest with a fresh nonce, keeping the - /// verifier output in /run for debugging. Fails on any non-zero exit — - /// including a GPU that cannot produce an attestation report at all (a - /// non-CC GPU, or CC mode left off by the host). - pub(super) async fn attest_gpu() -> Result<()> { - if !Path::new(NVATTEST).exists() { + /// Read the CC and DevTools state through NVML for every expected GPU. + /// NVML exposes these settings as system values through Device methods; + /// call them for every handle so every expected device must be enumerable. + pub(super) fn query_gpu_state(expected_devices: u32) -> Result<GpuState> { + let nvml = init_nvml(expected_devices)?; + let mut devices = Vec::with_capacity(expected_devices as usize); + for index in 0..expected_devices { + let device = nvml + .device_by_index(index) + .with_context(|| format!("failed to get NVML GPU at index {index}"))?; + let cc_enabled = device + .is_cc_enabled() + .with_context(|| format!("failed to query CC mode for GPU at index {index}"))?; + let devtools = device.is_cc_dev_mode_enabled().with_context(|| { + format!("failed to query DevTools mode for GPU at index {index}") + })?; + devices.push(GpuDeviceState { + cc_enabled, + devtools, + }); + } + Ok(GpuState { nvml, devices }) + } + + /// Set the system-wide GPU ready state without appraisal for the explicit + /// `gpu_policy.attest_gpu: false` compatibility path. + pub(super) fn set_gpu_ready_state(expected_devices: u32) -> Result<()> { + let nvml = init_nvml(expected_devices)?; + set_gpu_ready_state_with_nvml(&nvml) + } + + fn validate_attestation_output( + stdout: &[u8], + nonce: &str, + expected_devices: u32, + ) -> Result<ValidatedClaims> { + let output: NvattestOutput = + serde_json::from_slice(stdout).context("failed to parse nvattest JSON output")?; + if output.result_code != 0 { + bail!( + "nvattest JSON result is not successful (result_code={})", + output.result_code + ); + } + if output.claims.len() != expected_devices as usize { + bail!( + "gpu attestation count mismatch: expected {expected_devices}, got {}", + output.claims.len() + ); + } + let mut parsed_claims = Vec::with_capacity(output.claims.len()); + for (index, claim) in output.claims.iter().enumerate() { + let claim: NvidiaGpuClaim = serde_json::from_value(claim.clone()) + .with_context(|| format!("invalid GPU claim at index {index}"))?; + if claim.device_type != "gpu" { + bail!("gpu claim at index {index} has an invalid device type"); + } + if claim.eat_nonce != nonce || !claim.nonce_match { + bail!("gpu claim at index {index} has an invalid nonce"); + } + parsed_claims.push(claim); + } + Ok(ValidatedClaims { + raw: output.claims, + parsed: parsed_claims, + }) + } + + fn verify_claim_policy( + claims: &[NvidiaGpuClaim], + devices: &[GpuDeviceState], + policy: &GpuPolicy, + ) -> Result<()> { + for (index, device) in devices.iter().enumerate() { + if !device.cc_enabled { + bail!("gpu at index {index} does not enable confidential compute mode"); + } + if device.devtools && !policy.allow_devtools { + bail!("gpu at index {index} enables NVIDIA DevTools mode"); + } + } + for (index, claim) in claims.iter().enumerate() { + if claim.measres != "success" { + bail!("gpu claim at index {index} has unsuccessful measurements"); + } + if !policy.allow_insecure_boot && !claim.secboot { + bail!("gpu claim at index {index} does not assert secure boot"); + } + if !policy.allow_debug && claim.dbgstat != NvidiaGpuDebugStatus::Disabled { + bail!("gpu claim at index {index} does not disable debug mode"); + } + } + Ok(()) + } + + fn attestation_event(stdout: &[u8], devices: u32, devtools: bool) -> Result<Vec<u8>> { + let event = GpuAttestationEvent { + version: EVENT_VERSION, + provider: "nvidia", + devices, + cc_mode: "on", + devtools, + evidence_sha256: hex::encode(sha256(stdout)), + }; + serde_json::to_vec(&event).context("failed to serialize GPU attestation event") + } + + /// Run local GPU attestation via nvattest with a fresh evidence nonce. If + /// sys-config selects a collateral proxy, both RIM and OCSP traffic is + /// routed through it and NVIDIA's Trust Outpost policy accepts cached OCSP + /// responses whose responder nonce no longer matches. The independent GPU + /// evidence nonce remains mandatory and is checked below. + pub(super) async fn attest_gpu( + expected_devices: u32, + proxy_url: Option<&str>, + ) -> Result<GpuAttestationResult> { + if !nvattest::available() { bail!("nvattest is not available in this image"); } // Certificate/OCSP validation needs a sane clock even when @@ -1155,39 +1620,72 @@ mod gpu { if let Err(err) = cmd!(chronyc makestep) { warn!("failed to step system clock: {err:?}"); } - let nonce = hex::encode(rand::thread_rng().gen::<[u8; 32]>()); - let output = run_command( - NVATTEST, - &[ - "attest", - "--device", - "gpu", - "--verifier", - "local", - "--nonce", - &nonce, - ], - ATTESTATION_TIMEOUT, - ) - .await?; - if !output.stderr.is_empty() { - info!("nvattest: {}", truncated_lossy(&output.stderr, 2048)); - } - if !output.status.success() { - bail!( - "nvattest exited with {}: {}", - output.status, - truncated_lossy(&output.stderr, 512), - ); - } - if let Err(err) = save_attestation_output(&output.stdout) { - warn!("failed to save GPU attestation output: {err:?}"); + let nonce: [u8; nvattest::NONCE_LEN] = rand::thread_rng().gen(); + let (nonce, output) = nvattest::run(&nonce, proxy_url, nvattest::DEFAULT_TIMEOUT).await?; + // Persist before judging the exit status: a failed appraisal is exactly + // when the evidence is worth having on disk. + save_attestation_output(&output.stdout).context("failed to save GPU attestation output")?; + nvattest::check_status(&output)?; + let claims = validate_attestation_output(&output.stdout, &nonce, expected_devices)?; + Ok(GpuAttestationResult { + claims: claims.raw, + parsed_claims: claims.parsed, + output: output.stdout, + devices: expected_devices, + }) + } + + pub(super) fn measure_gpu_policy(compose_path: &Path) -> Result<[u8; 32]> { + let compose_json = fs::read(compose_path) + .with_context(|| format!("failed to read {}", compose_path.display()))?; + let digest = gpu_policy_hash(&compose_json).context("failed to hash raw GPU policy")?; + emit_runtime_event("gpu-policy-hash", &digest) + .context("failed to emit GPU policy measurement")?; + Ok(digest) + } + + pub(super) fn evaluate_rego_policy(policy: &GpuPolicy, claims: &[Value]) -> Result<()> { + let Some(rego) = policy.rego.as_deref() else { + return Ok(()); + }; + evaluate_policy(rego, claims).context("failed to apply GPU Rego policy") + } + + /// Evaluate the app-provided Rego v0 policy using the same input shape as + /// NVIDIA relying-party policies: the nvattest `claims` JSON array. + pub(super) fn evaluate_policy(policy: &str, claims: &[Value]) -> Result<()> { + evaluate_policy_with_timeout(policy, claims, POLICY_TIMEOUT) + } + + fn evaluate_policy_with_timeout( + policy: &str, + claims: &[Value], + timeout: Duration, + ) -> Result<()> { + let mut engine = regorus::Engine::new(); + engine.set_rego_v0(true); + engine.set_execution_timer_config(regorus::utils::limits::ExecutionTimerConfig { + limit: timeout, + check_interval: std::num::NonZeroU32::new(1024).unwrap_or(std::num::NonZeroU32::MIN), + }); + engine + .add_policy("gpu-policy.rego".to_string(), policy.to_string()) + .context("failed to load GPU policy")?; + let input = serde_json::to_string(claims).context("failed to serialize GPU claims")?; + engine + .set_input_json(&input) + .context("failed to set GPU policy input")?; + if !engine + .eval_bool_query(POLICY_ENTRYPOINT.to_string(), false) + .context("failed to evaluate GPU policy")? + { + bail!("gpu policy rejected the attestation claims"); } Ok(()) } fn save_attestation_output(stdout: &[u8]) -> Result<()> { - let output_path = Path::new(ATTESTATION_OUTPUT); + let output_path = Path::new(GPU_ATTESTATION_OUTPUT); if let Some(parent) = output_path.parent() { fs::create_dir_all(parent)?; } @@ -1199,54 +1697,513 @@ mod gpu { Ok(()) } - fn truncated_lossy(bytes: &[u8], limit: usize) -> String { - let text = String::from_utf8_lossy(bytes); - let text = text.trim(); - match text.char_indices().nth(limit) { - Some((idx, _)) => format!("{}...", &text[..idx]), - None => text.to_string(), + #[cfg(test)] + mod tests { + use super::*; + + fn nvattest_output(nonce: &str, claims: usize) -> Vec<u8> { + let claims = (0..claims) + .map(|_| { + serde_json::json!({ + "x-nvidia-device-type": "gpu", + "eat_nonce": nonce, + "x-nvidia-gpu-attestation-report-nonce-match": true, + "measres": "success", + "secboot": true, + "dbgstat": "disabled" + }) + }) + .collect::<Vec<_>>(); + serde_json::to_vec(&serde_json::json!({ + "result_code": 0, + "claims": claims, + "detached_eat": {} + })) + .unwrap() + } + + // Captured with the pinned nvattest SDK on an Ubuntu 22.04 GCP A3 TDX + // VM with an H100 and the NVIDIA 580 open kernel driver. + const H100_ATTESTATION_OUTPUT: &[u8] = + include_bytes!("../tests/fixtures/gpu_attestation_h100.json"); + + /// The `AttestGpu` API documents that its output cannot be verified by a + /// third party, because the local verifier reports a conclusion rather + /// than the GPU's signed report. That is a claim about NVIDIA's output + /// format, so pin it: if a future SDK starts signing the detached EAT, + /// this fails and the API docs need revisiting rather than quietly + /// becoming wrong. + #[test] + fn local_verifier_output_is_unsigned_self_report() { + let output: Value = serde_json::from_slice(H100_ATTESTATION_OUTPUT).unwrap(); + let eat = &output["detached_eat"]; + let jwt = eat[0][1].as_str().expect("detached EAT carries a JWT"); + let (header_b64, rest) = jwt.split_once('.').unwrap(); + let (_, signature) = rest.split_once('.').unwrap(); + assert!( + signature.is_empty(), + "detached EAT is signed; AttestGpu docs claim it is not" + ); + + use base64::Engine as _; + let header = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(header_b64) + .unwrap(); + let header: Value = serde_json::from_slice(&header).unwrap(); + assert_eq!(header["alg"], "none"); + + // And the signed artifacts really are absent: the claims carry + // verdicts about the certificate chain, not the chain itself. + let claim = &output["claims"][0]; + assert_eq!( + claim["x-nvidia-gpu-attestation-report-signature-verified"], + true + ); + assert!( + claim["x-nvidia-gpu-attestation-report-cert-chain"] + .as_object() + .expect("cert-chain claim is a verdict object") + .keys() + .all(|key| key.starts_with("x-nvidia-cert-")), + "cert-chain claim carries certificates, not just verdicts" + ); + } + + #[test] + fn gpu_count_rejects_non_nvidia_gpus() { + let mixed = GpuInventory { + total: 2, + nvidia: 1, + }; + assert!(nvidia_gpu_count(mixed).is_err()); + + let nvidia = GpuInventory { + total: 2, + nvidia: 2, + }; + assert_eq!(nvidia_gpu_count(nvidia).unwrap(), 2); + } + + #[test] + fn basic_policy_requires_cc_and_rejects_devtools_by_default() { + let nonce = "44".repeat(32); + let output = nvattest_output(&nonce, 1); + let claims = validate_attestation_output(&output, &nonce, 1).unwrap(); + let production = [GpuDeviceState { + cc_enabled: true, + devtools: false, + }]; + verify_claim_policy(&claims.parsed, &production, &GpuPolicy::default()).unwrap(); + + let non_cc = [GpuDeviceState { + cc_enabled: false, + devtools: false, + }]; + let err = + verify_claim_policy(&claims.parsed, &non_cc, &GpuPolicy::default()).unwrap_err(); + assert!(err.to_string().contains("confidential compute mode")); + + let devtools = [GpuDeviceState { + cc_enabled: true, + devtools: true, + }]; + assert!(verify_claim_policy(&claims.parsed, &devtools, &GpuPolicy::default()).is_err()); + verify_claim_policy( + &claims.parsed, + &devtools, + &GpuPolicy { + allow_devtools: true, + ..Default::default() + }, + ) + .unwrap(); + } + + #[test] + fn nvattest_output_requires_every_expected_gpu_and_fresh_nonce() { + let nonce = "11".repeat(32); + let valid = nvattest_output(&nonce, 2); + validate_attestation_output(&valid, &nonce, 2).unwrap(); + assert!(validate_attestation_output(&valid, &nonce, 1).is_err()); + + let mut invalid: Value = serde_json::from_slice(&valid).unwrap(); + invalid["claims"][1]["eat_nonce"] = Value::String("stale".to_string()); + assert!( + validate_attestation_output(&serde_json::to_vec(&invalid).unwrap(), &nonce, 2) + .is_err() + ); + + // Basic claim settings are enforced after structural validation. + let mut extra_claims: Value = serde_json::from_slice(&valid).unwrap(); + extra_claims["claims"][0]["dbgstat"] = Value::String("enabled".to_string()); + validate_attestation_output(&serde_json::to_vec(&extra_claims).unwrap(), &nonce, 2) + .unwrap(); + } + + #[test] + fn basic_claim_policy_is_fail_closed_and_honors_opt_ins() { + let nonce = "33".repeat(32); + let output = nvattest_output(&nonce, 1); + let claims = validate_attestation_output(&output, &nonce, 1).unwrap(); + let devices = [GpuDeviceState { + cc_enabled: true, + devtools: false, + }]; + verify_claim_policy(&claims.parsed, &devices, &GpuPolicy::default()).unwrap(); + + let with_policy = |name: &str, value: Value, policy: &GpuPolicy| { + let mut output: Value = serde_json::from_slice(&output).unwrap(); + output["claims"][0][name] = value; + let output = serde_json::to_vec(&output).unwrap(); + let claims = validate_attestation_output(&output, &nonce, 1).unwrap(); + verify_claim_policy(&claims.parsed, &devices, policy) + }; + + assert!(with_policy("secboot", Value::Bool(false), &GpuPolicy::default()).is_err()); + with_policy( + "secboot", + Value::Bool(false), + &GpuPolicy { + allow_insecure_boot: true, + ..Default::default() + }, + ) + .unwrap(); + + assert!(with_policy( + "dbgstat", + Value::String("enabled".to_string()), + &GpuPolicy::default(), + ) + .is_err()); + with_policy( + "dbgstat", + Value::String("enabled".to_string()), + &GpuPolicy { + allow_debug: true, + ..Default::default() + }, + ) + .unwrap(); + + let mut unknown_debug: Value = serde_json::from_slice(&output).unwrap(); + unknown_debug["claims"][0]["dbgstat"] = Value::String("unknown".to_string()); + assert!(validate_attestation_output( + &serde_json::to_vec(&unknown_debug).unwrap(), + &nonce, + 1, + ) + .is_err()); + + assert!(with_policy( + "measres", + Value::String("failure".to_string()), + &GpuPolicy { + allow_debug: true, + allow_insecure_boot: true, + ..Default::default() + }, + ) + .is_err()); + + for required in ["measres", "secboot", "dbgstat"] { + let mut missing: Value = serde_json::from_slice(&output).unwrap(); + missing["claims"][0] + .as_object_mut() + .unwrap() + .remove(required); + assert!(validate_attestation_output( + &serde_json::to_vec(&missing).unwrap(), + &nonce, + 1, + ) + .is_err()); + } + } + + #[test] + fn real_h100_attestation_fixture_validates_and_drives_rego() { + let nonce = "11".repeat(32); + let claims = validate_attestation_output(H100_ATTESTATION_OUTPUT, &nonce, 1).unwrap(); + assert_eq!(claims.raw[0]["hwmodel"], "GH100 A01 GSP BROM"); + assert_eq!(claims.raw[0]["x-nvidia-gpu-claims-version"], "3.0"); + verify_claim_policy( + &claims.parsed, + &[GpuDeviceState { + cc_enabled: true, + devtools: false, + }], + &GpuPolicy::default(), + ) + .unwrap(); + + let policy = r#" + package policy + default nv_match = false + nv_match { + count(input) == 1 + input[0].secboot == true + input[0].dbgstat == "disabled" + input[0].measres == "success" + } + "#; + evaluate_policy(policy, &claims.raw).unwrap(); + } + + #[test] + fn event_commits_to_complete_nvattest_output() { + let nonce = "22".repeat(32); + let output = nvattest_output(&nonce, 1); + let event: Value = + serde_json::from_slice(&attestation_event(&output, 1, true).unwrap()).unwrap(); + assert_eq!(event["version"], EVENT_VERSION); + assert_eq!(event["devices"], 1); + assert!(event.get("policy").is_none()); + assert_eq!(event["cc_mode"], "on"); + assert_eq!(event["devtools"], true); + assert_eq!(event["evidence_sha256"], hex::encode(sha256(&output))); + } + + #[test] + fn app_policy_receives_claims_array_and_must_return_true() { + let claims = vec![serde_json::json!({"status": "accepted"})]; + let policy = r#" + package policy + default nv_match = false + nv_match { + count(input) == 1 + input[0].status == "accepted" + } + "#; + evaluate_policy(policy, &claims).unwrap(); + assert!(evaluate_policy(policy, &[]).is_err()); + + let rejected = vec![serde_json::json!({"status": "rejected"})]; + assert!(evaluate_policy(policy, &rejected).is_err()); + assert!(evaluate_policy("package policy", &claims).is_err()); + assert!(evaluate_policy("not valid rego", &claims).is_err()); + + let allow_no_gpus = GpuPolicy { + rego: Some( + r#" + package policy + default nv_match = false + nv_match { count(input) == 0 } + "# + .to_string(), + ), + ..Default::default() + }; + evaluate_rego_policy(&allow_no_gpus, &[]).unwrap(); + + let require_one_gpu = GpuPolicy { + rego: Some(policy.to_string()), + ..Default::default() + }; + assert!(evaluate_rego_policy(&require_one_gpu, &[]).is_err()); + evaluate_rego_policy(&GpuPolicy::default(), &[]).unwrap(); + } + + #[test] + fn rego_policy_evaluation_is_time_bounded() { + let policy = r#" + package policy + default nv_match = false + nv_match { + count([x | + x := numbers.range(0, 5000)[_] + y := numbers.range(0, 5000)[_] + x == y + ]) > 0 + } + "#; + evaluate_policy_with_timeout(policy, &[], Duration::from_millis(50)).unwrap_err(); + } + + #[test] + fn gpu_policy_measurement_defaults_to_empty_object_and_uses_raw_json() { + let no_requirements = br#"{}"#; + let absent = br#"{"requirements": {}}"#; + let empty_digest = sha256(b"{}"); + assert_eq!(gpu_policy_hash(no_requirements).unwrap(), empty_digest); + assert_eq!(gpu_policy_hash(absent).unwrap(), empty_digest); + + let empty = br#"{"requirements": {"gpu_policy": {}}}"#; + assert_eq!(gpu_policy_hash(empty).unwrap(), empty_digest); + + let explicit_default = br#"{"requirements":{"gpu_policy":{"attest_gpu":true}}}"#; + let explicit_default_digest = gpu_policy_hash(explicit_default).unwrap(); + assert_ne!(explicit_default_digest, empty_digest); + + let reordered = br#" + { + "requirements": { + "gpu_policy": { + "rego": "package policy", + "allow_debug": false + } + } + } + "#; + let canonical_order = + br#"{"requirements":{"gpu_policy":{"allow_debug":false,"rego":"package policy"}}}"#; + assert_eq!( + gpu_policy_hash(reordered).unwrap(), + gpu_policy_hash(canonical_order).unwrap() + ); } } } impl Stage0<'_> { - /// Enforce `requirements.attest_gpu` (default true): attest an attached - /// NVIDIA GPU before continuing to key provisioning, or — when explicitly - /// disabled — set the GPU ready state without verification. - async fn setup_gpu(&self) -> Result<()> { - if !gpu::nvidia_gpu_present()? { - return Ok(()); - } - if !self.shared.app_compose.attest_gpu() { - warn!("requirements.attest_gpu is false; setting GPU ready state without attestation"); + /// Enforce `requirements.gpu_policy.attest_gpu` (default true): attest an + /// attached NVIDIA GPU before continuing to key provisioning, or — when + /// explicitly disabled — set the GPU ready state without verification. The + /// optional Rego policy is always evaluated; when no attestation is + /// performed, its claims-array input is empty. + async fn measure_gpu(&self) -> Result<[u8; 32]> { + let gpu_policy_hash = gpu::measure_gpu_policy(&self.shared.dir.app_compose_file())?; + + let gpu_policy = self + .shared + .app_compose + .requirements + .as_ref() + .map(|requirements| requirements.gpu_policy.clone()) + .unwrap_or_default(); + + let inventory = gpu::gpu_inventory()?; + if !gpu_policy.attest_gpu { + // Attestation is explicitly disabled, so there are no claims. Rego + // still runs with an empty input before any GPU is made ready. + gpu::evaluate_rego_policy(&gpu_policy, &[])?; + if gpu_policy.rego.is_some() { + info!("application GPU Rego policy accepted an empty claims array"); + } + if inventory.nvidia == 0 { + return Ok(gpu_policy_hash); + } + warn!( + "requirements.gpu_policy.attest_gpu is false; setting GPU ready state without attestation" + ); // Best-effort: a GPU with CC mode off has no ready state to set. - if let Err(err) = gpu::set_gpu_ready_state().await { + if let Err(err) = gpu::set_gpu_ready_state(inventory.nvidia) { warn!("failed to set GPU ready state: {err:?}"); } - return Ok(()); + return Ok(gpu_policy_hash); + } + let expected_devices = gpu::nvidia_gpu_count(inventory)?; + if expected_devices == 0 { + gpu::evaluate_rego_policy(&gpu_policy, &[])?; + if gpu_policy.rego.is_some() { + info!("application GPU Rego policy accepted an empty claims array"); + } + return Ok(gpu_policy_hash); } self.vmm.notify_q("boot.progress", "attesting GPU").await; info!("verifying GPU TEE attestation"); - gpu::attest_gpu().await?; - gpu::set_gpu_ready_state().await?; + let attestation = gpu::attest_gpu( + expected_devices, + self.shared + .sys_config + .nvidia_attestation_proxy_url + .as_deref(), + ) + .await?; + + let gpu_state = gpu::query_gpu_state(expected_devices)?; + attestation + .verify_claim_policy(&gpu_state, &gpu_policy) + .context("failed to apply basic GPU policy")?; + gpu::evaluate_rego_policy(&gpu_policy, attestation.claims())?; + + info!("application GPU policy accepted the attestation claims and state"); + gpu_state.set_ready()?; + let devtools = gpu_state.any_devtools(); + let event = attestation.event(devtools)?; + emit_runtime_event("gpu-attestation", &event) + .context("failed to emit GPU attestation event")?; info!("GPU TEE attestation succeeded"); + Ok(gpu_policy_hash) + } +} + +/// Owns the inputs needed to (re)register this CVM with dstack-gateway. +/// +/// Loading is separated from refreshing so a long-running caller (the gateway +/// checker) can pay the parsing cost once and then refresh repeatedly. +pub struct GatewayRefresher { + shared: HostShared, + keys: AppKeys, +} + +impl GatewayRefresher { + /// Load the host-shared config and app keys from `work_dir`. + pub fn load(work_dir: &Path) -> Result<Self> { + let host_shared_dir = work_dir.join(HOST_SHARED_DIR_NAME); + let shared = HostShared::load(host_shared_dir.as_path()).with_context(|| { + format!( + "Failed to load host-shared dir: {}", + host_shared_dir.display() + ) + })?; + let keys_path = shared.dir.join(APP_KEYS); + let keys: AppKeys = deserialize_json_file(&keys_path) + .with_context(|| format!("Failed to load app keys from {}", keys_path.display()))?; + Ok(Self { shared, keys }) + } + + /// Whether this app opted into dstack-gateway at all. + pub fn gateway_enabled(&self) -> bool { + self.shared.app_compose.gateway_enabled() + } + + /// Validate the parts of the gateway config that can never become valid by + /// waiting. These are deployment mistakes, not outages, so callers that + /// retry should give up instead of looping forever. + pub fn check_config(&self) -> Result<()> { + if self.keys.gateway_app_id.is_empty() { + bail!("Missing allowed dstack-gateway app id"); + } + if self.shared.sys_config.gateway_urls.is_empty() + && self.shared.sys_config.gateway_clusters.is_empty() + { + bail!("Missing gateway urls"); + } Ok(()) } + + /// Register with dstack-gateway and apply the returned WireGuard config. + pub async fn refresh(&self, force: bool) -> Result<()> { + GatewayContext::new(&self.shared, &self.keys) + .setup(force) + .await + } } pub async fn cmd_gateway_refresh(args: GatewayRefreshArgs) -> Result<()> { - let host_shared_dir = args.work_dir.join(HOST_SHARED_DIR_NAME); - let shared = HostShared::load(host_shared_dir.as_path()).with_context(|| { - format!( - "Failed to load host-shared dir: {}", - host_shared_dir.display() - ) - })?; - let keys_path = shared.dir.join(APP_KEYS); - let keys: AppKeys = deserialize_json_file(&keys_path) - .with_context(|| format!("Failed to load app keys from {}", keys_path.display()))?; + GatewayRefresher::load(&args.work_dir)? + .refresh(args.force) + .await +} - GatewayContext::new(&shared, &keys).setup(args.force).await +/// Accept only a certificate the KMS issued for its own RPC endpoint. +/// +/// The attestation behind this certificate is already verified by the RA-TLS +/// layer, and the KMS identity that matters to the guest is its CA public key, +/// pinned separately by `verify_key_provider_id`. All that is left here is +/// refusing a certificate minted for some other purpose. +fn validate_kms_rpc_cert(cert: Option<CertInfo>) -> Result<()> { + let Some(cert) = cert else { + bail!("missing server cert"); + }; + let Some(usage) = cert.special_usage else { + bail!("missing server cert usage"); + }; + if usage != "kms:rpc" { + bail!("Invalid server cert usage: {usage}"); + } + Ok(()) } struct AppIdValidator { @@ -1279,6 +2236,8 @@ impl AppIdValidator { struct AppInfo { instance_info: InstanceInfo, compose_hash: [u8; 32], + gpu_policy_hash: [u8; 32], + init_script_hashes: Vec<Vec<u8>>, } struct Stage0<'a> { @@ -1294,11 +2253,27 @@ struct Stage1<'a> { keys: AppKeys, } +fn validate_key_provider_inputs(kind: KeyProviderKind, kms_urls: &[String]) -> Result<()> { + if kind.is_kms() && kms_urls.is_empty() { + bail!("No KMS URLs are set"); + } + Ok(()) +} + +fn kms_rpc_url(base: &str) -> String { + let base = base.trim_end_matches('/'); + if base.ends_with("/prpc") { + base.to_string() + } else { + format!("{base}/prpc") + } +} + impl<'a> Stage0<'a> { fn host_api(&self) -> HostApi { HostApi::new( self.shared.sys_config.host_api_url.clone(), - self.shared.sys_config.pccs_url.clone(), + self.shared.sys_config.collateral_urls().pccs, ) } fn load(args: &'a SetupArgs) -> Result<Self> { @@ -1314,7 +2289,7 @@ impl<'a> Stage0<'a> { let host_shared = HostShared::copy("/tmp/.host-shared".as_ref(), &host_shared_copy_dir)?; let host_api = HostApi::new( host_shared.sys_config.host_api_url.clone(), - host_shared.sys_config.pccs_url.clone(), + host_shared.sys_config.collateral_urls().pccs, ); Ok(Self { args, @@ -1339,6 +2314,7 @@ impl<'a> Stage0<'a> { .context("Failed to get temp ca cert")? }; let cert_pair = generate_ra_cert(tmp_ca.temp_ca_cert.clone(), tmp_ca.temp_ca_key.clone())?; + let attestation_verifier = attestation_verifier(&self.shared.sys_config)?; let ra_client = RaClientConfig::builder() .tls_no_check(false) .tls_built_in_root_certs(false) @@ -1346,29 +2322,8 @@ impl<'a> Stage0<'a> { .tls_client_cert(cert_pair.cert_pem) .tls_client_key(cert_pair.key_pem) .tls_ca_cert(tmp_ca.ca_cert.clone()) - .maybe_pccs_url(self.shared.sys_config.pccs_url.clone()) - .cert_validator(Box::new(|cert| { - let Some(cert) = cert else { - bail!("Missing server cert"); - }; - let Some(usage) = cert.special_usage else { - bail!("Missing server cert usage"); - }; - if usage != "kms:rpc" { - bail!("Invalid server cert usage: {usage}"); - } - if let Some(att) = &cert.attestation { - match att.decode_app_info(false) { - Ok(kms_info) => emit_runtime_event("mr-kms", &kms_info.mr_aggregated) - .context("failed to extend mr-kms to the launch measurement")?, - Err(err) if is_unsupported_app_info_quote(&err) => { - warn!("Skipping mr-kms runtime event for unsupported attestation quote: {err:#}"); - } - Err(err) => return Err(err).context("Failed to decode app_info"), - } - } - Ok(()) - })) + .attestation_verifier(attestation_verifier) + .cert_validator(Box::new(validate_kms_rpc_cert)) .build() .into_client() .context("Failed to create client")?; @@ -1413,7 +2368,7 @@ impl<'a> Stage0<'a> { let keys = 'out: { let mut error = anyhow!("unknown error"); for (i, kms_url) in self.shared.sys_config.kms_urls.iter().enumerate() { - let kms_url = format!("{kms_url}/prpc"); + let kms_url = kms_rpc_url(kms_url); let response = self.request_app_keys_from_kms_url(kms_url.clone()).await; match response { Ok(response) => { @@ -1503,6 +2458,7 @@ impl<'a> Stage0<'a> { async fn request_app_keys(&self) -> Result<AppKeys> { let key_provider = self.shared.app_compose.key_provider(); + validate_key_provider_inputs(key_provider, &self.shared.sys_config.kms_urls)?; match key_provider { KeyProviderKind::Kms => self.request_app_keys_from_kms().await, KeyProviderKind::Local => self.get_keys_from_local_key_provider().await, @@ -1680,7 +2636,7 @@ impl<'a> Stage0<'a> { if opts.storage_encrypted { info!("Setting up disk encryption"); - self.luks_setup(disk_crypt_key, name)?; + self.luks_setup(disk_crypt_key, name, opts.storage_discard)?; } else { info!("Skipping disk encryption as requested by kernel cmdline"); } @@ -1688,19 +2644,23 @@ impl<'a> Stage0<'a> { match opts.storage_fs { FsType::Zfs => { info!("Creating ZFS filesystem"); + let autotrim = if opts.storage_discard { "on" } else { "off" }; cmd! { - zpool create -o autoexpand=on dstack $fs_dev; + zpool create -o autoexpand=on -o autotrim=$autotrim -m none dstack $fs_dev; zfs create -o mountpoint=$mount_point -o atime=off -o checksum=blake3 dstack/data; } .context("Failed to create zpool")?; } FsType::Ext4 => { info!("Creating ext4 filesystem"); - cmd! { - mkfs.ext4 -F $fs_dev; - mount $fs_dev $mount_point; + cmd!(mkfs.ext4 -F $fs_dev).context("Failed to create ext4 filesystem")?; + if opts.storage_discard { + cmd!(mount -o discard $fs_dev $mount_point) + .context("failed to mount ext4 filesystem with discard")?; + } else { + cmd!(mount $fs_dev $mount_point) + .context("failed to mount ext4 filesystem")?; } - .context("Failed to create ext4 filesystem")?; } } } else { @@ -1710,7 +2670,7 @@ impl<'a> Stage0<'a> { if opts.storage_encrypted { info!("Mounting encrypted data disk"); - self.open_encrypted_volume(disk_crypt_key, name)?; + self.open_encrypted_volume(disk_crypt_key, name, opts.storage_discard)?; } else { info!("Mounting unencrypted data disk"); } @@ -1719,16 +2679,32 @@ impl<'a> Stage0<'a> { FsType::Zfs => { cmd! { zpool import dstack; + } + .context("Failed to import zpool")?; + let previous_autotrim = cmd!(zpool get -H -o value autotrim dstack) + .map(|value| value.trim().to_owned()) + .unwrap_or_default(); + let autotrim = if opts.storage_discard { "on" } else { "off" }; + cmd! { + zpool set autotrim=$autotrim dstack; zpool status dstack; zpool online -e dstack $fs_dev; // triggers autoexpand } - .context("Failed to import zpool")?; + .context("Failed to configure zpool")?; + if opts.storage_discard && previous_autotrim == "off" { + // autotrim only covers future frees. Start an asynchronous + // trim on first upgrade so historical free space is returned + // without delaying boot. + if let Err(err) = cmd!(zpool trim dstack) { + warn!("failed to start initial zpool trim: {err}"); + } + } if cmd!(mountpoint -q $mount_point).is_err() { cmd!(zfs mount dstack/data).context("Failed to mount zpool")?; } } FsType::Ext4 => { - Self::mount_e2fs(&fs_dev, mount_point) + Self::mount_e2fs(&fs_dev, mount_point, opts.storage_discard) .context("Failed to mount ext4 filesystem")?; } } @@ -1736,7 +2712,11 @@ impl<'a> Stage0<'a> { Ok(()) } - fn mount_e2fs(dev: &impl AsRef<Path>, mount_point: &impl AsRef<Path>) -> Result<()> { + fn mount_e2fs( + dev: &impl AsRef<Path>, + mount_point: &impl AsRef<Path>, + discard: bool, + ) -> Result<()> { let dev = dev.as_ref(); let mount_point = mount_point.as_ref(); info!("Checking filesystem"); @@ -1764,36 +2744,62 @@ impl<'a> Stage0<'a> { } } - cmd! { - info "Trying to resize filesystem if needed"; - resize2fs $dev; - info "Mounting filesystem"; - mount $dev $mount_point; + if discard { + cmd! { + resize2fs $dev; + mount -o discard $dev $mount_point; + } + .context("failed to resize and mount ext4 filesystem with discard")?; + } else { + cmd! { + resize2fs $dev; + mount $dev $mount_point; + } + .context("failed to resize and mount ext4 filesystem")?; } - .context("Failed to prepare ext4 filesystem")?; Ok(()) } - fn luks_setup(&self, disk_crypt_key: &str, name: &str) -> Result<()> { + fn luks_setup(&self, disk_crypt_key: &str, name: &str, discard: bool) -> Result<()> { let root_hd = &self.args.device; let sector_offset = PAYLOAD_OFFSET / 512; - cmd! { - info "Formatting encrypted disk"; - echo -n $disk_crypt_key | - cryptsetup luksFormat - --type luks2 - --offset $sector_offset - --cipher aes-xts-plain64 - --pbkdf pbkdf2 - -d- - $root_hd - $name; - } - .or(Err(anyhow!("Failed to setup luks volume")))?; - self.open_encrypted_volume(disk_crypt_key, name) - } - - fn open_encrypted_volume(&self, disk_crypt_key: &str, name: &str) -> Result<()> { + info!("Formatting encrypted disk"); + let sector_offset = sector_offset.to_string(); + let mut child = Command::new("cryptsetup") + .args([ + "luksFormat", + "--type", + "luks2", + "--offset", + §or_offset, + "--cipher", + "aes-xts-plain64", + "--pbkdf", + "pbkdf2", + "-d-", + ]) + .arg(root_hd) + .arg(name) + .stdin(Stdio::piped()) + .spawn() + .context("Failed to start cryptsetup luksFormat")?; + child + .stdin + .take() + .context("cryptsetup stdin is unavailable")? + .write_all(disk_crypt_key.as_bytes()) + .context("Failed to send key to cryptsetup luksFormat")?; + if !child + .wait() + .context("Failed to wait for cryptsetup luksFormat")? + .success() + { + bail!("Failed to setup luks volume"); + } + self.open_encrypted_volume(disk_crypt_key, name, discard) + } + + fn open_encrypted_volume(&self, disk_crypt_key: &str, name: &str, discard: bool) -> Result<()> { let root_hd = &self.args.device; let disk_crypt_key = disk_crypt_key.trim(); // Create a private tmpfs mount to ensure the header stays in-memory. @@ -1820,11 +2826,34 @@ impl<'a> Stage0<'a> { let hdr_file = fs::File::open(&in_mem_hdr).context("Failed to open LUKS2 header")?; validate_luks2_headers(hdr_file).context("Failed to validate LUKS2 header")?; - cmd! { - info "Opening the device"; - echo -n $disk_crypt_key | cryptsetup luksOpen --type luks2 --header $in_mem_hdr -d- $root_hd $name; + info!("Opening the device"); + let mut command = Command::new("cryptsetup"); + command.args(["luksOpen", "--type", "luks2"]); + if discard { + command.arg("--allow-discards"); + } + let mut child = command + .arg("--header") + .arg(&in_mem_hdr) + .arg("-d-") + .arg(root_hd) + .arg(name) + .stdin(Stdio::piped()) + .spawn() + .context("Failed to start cryptsetup luksOpen")?; + child + .stdin + .take() + .context("cryptsetup stdin is unavailable")? + .write_all(disk_crypt_key.as_bytes()) + .context("Failed to send key to cryptsetup luksOpen")?; + if !child + .wait() + .context("Failed to wait for cryptsetup luksOpen")? + .success() + { + bail!("Failed to open encrypted data disk"); } - .or(Err(anyhow!("Failed to open encrypted data disk")))?; // Wait for device mapper to create the device let dm_path = format!("/dev/mapper/{name}"); @@ -1842,13 +2871,13 @@ impl<'a> Stage0<'a> { Ok(()) } - fn measure_app_info(&self) -> Result<AppInfo> { + async fn measure_app_info(&self) -> Result<AppInfo> { let compose_hash = sha256_file(self.shared.dir.app_compose_file())?; let truncated_compose_hash = truncate(&compose_hash, 20); let key_provider = self.shared.app_compose.key_provider(); let mut instance_info = self.shared.instance_info.clone(); - let is_snp = AttestationMode::detect() - .map(|mode| mode == AttestationMode::DstackAmdSevSnp) + let is_snp = detect_tee_variant() + .map(|mode| mode == TeeVariant::DstackAmdSevSnp) .unwrap_or(false); if instance_info.app_id.is_empty() { @@ -1894,6 +2923,21 @@ impl<'a> Stage0<'a> { emit_runtime_event("system-preparing", &[])?; emit_runtime_event("app-id", &instance_info.app_id)?; emit_runtime_event("compose-hash", &compose_hash)?; + let init_script_hashes: Vec<Vec<u8>> = self + .shared + .app_compose + .init_script + .iter() + .map(|script| sha256(script.as_bytes()).to_vec()) + .collect(); + for script_hash in &init_script_hashes { + emit_runtime_event("init-script-hash", script_hash)?; + } + let gpu_policy_hash = self + .measure_gpu() + .await + .context("failed to verify GPU TEE attestation")?; + emit_runtime_event("instance-id", &instance_id)?; emit_runtime_event("boot-mr-done", &[])?; @@ -1921,32 +2965,27 @@ impl<'a> Stage0<'a> { Ok(AppInfo { instance_info, compose_hash, + gpu_policy_hash, + init_script_hashes, }) } fn verify_app(&self, app_info: &AppInfo, keys: &AppKeys) -> Result<()> { - if dstack_attest::attestation::AttestationMode::detect().is_err() { - // DEV-ONLY fallback: no TEE hardware present (e.g. a local no_tee - // dev CVM). There is no TDX/SEV-SNP quote to read an - // mr_config_id from, so skip this check instead of failing boot. - // Real TEE hosts always resolve an AttestationMode and take the - // unchanged verification path below. - warn!("no TEE attestation mode available; skipping mr_config_id verification (no_tee dev mode)"); - } else { - config_id_verifier::verify_mr_config_id( - &app_info.compose_hash, - &app_info - .instance_info - .app_id - .as_slice() - .try_into() - .ok() - .context("Invalid app id")?, - &app_info.instance_info.instance_id, - keys.key_provider.kind(), - keys.key_provider.id(), - )?; - } + config_id_verifier::verify_mr_config_id( + &app_info.compose_hash, + &app_info.gpu_policy_hash, + &app_info.init_script_hashes, + &app_info + .instance_info + .app_id + .as_slice() + .try_into() + .ok() + .context("Invalid app id")?, + &app_info.instance_info.instance_id, + keys.key_provider.kind(), + keys.key_provider.id(), + )?; self.verify_key_provider_id(keys.key_provider.id())?; // TPM uses an empty id: the instance app-root pubkey is not a stable // provider identity and must not enter the launch measurement chain. @@ -1967,6 +3006,7 @@ impl<'a> Stage0<'a> { async fn setup_fs(self) -> Result<Stage1<'a>> { let app_info = self .measure_app_info() + .await .context("Failed to measure app info")?; if self.shared.app_compose.key_provider().is_kms() { cmd_show_mrs()?; @@ -2159,9 +3199,27 @@ impl Stage1<'_> { self.vmm .notify_q("boot.progress", "setting up dstack-gateway") .await; - GatewayContext::new(&self.shared, &self.keys) + if let Err(error) = GatewayContext::new(&self.shared, &self.keys) .setup(true) - .await?; + .await + { + warn!( + "dstack-gateway registration is unavailable during boot; continuing without a route: {error:#}" + ); + // Boot no longer fails here, so a guest log line would be the only + // trace of it: the VM would report a clean boot while having no + // ingress at all. Report it to the host so the degraded state is + // visible from the VMM. The gateway checker clears this once it + // manages to register. + self.vmm + .notify_q( + "boot.error", + &format!( + "dstack-gateway registration failed, the app has no ingress route: {error:#}" + ), + ) + .await; + } self.vmm .notify_q("boot.progress", "setting up docker") .await; @@ -2192,7 +3250,6 @@ impl Stage1<'_> { let config = serde_json::json!({ "default": { "core": { - "pccs_url": self.shared.sys_config.pccs_url, "data_disks": data_disks, } } @@ -2362,6 +3419,16 @@ fn validate_single_luks2_header(mut reader: impl std::io::Read, hdr_ind: u64) -> if area.encryption() != "aes-xts-plain64" { bail!("Invalid LUKS keyslot encryption: {}", area.encryption()); } + // Pin where the encrypted key material is read from. The binary area + // must sit between the two header copies and the encrypted payload; + // otherwise a host with raw disk access could redirect it elsewhere. + if area.offset() < 2 * hdr_size || area.offset() + area.size() > PAYLOAD_OFFSET { + bail!( + "Invalid LUKS keyslot area: offset={} size={}", + area.offset(), + area.size() + ); + } if *key_size != 64 { bail!("Invalid LUKS keyslot key size: {key_size}"); } @@ -2372,6 +3439,9 @@ fn validate_single_luks2_header(mut reader: impl std::io::Read, hdr_ind: u64) -> let LuksKdf::pbkdf2 { hash, iterations: _, + // Salts are left unchecked on purpose: the passphrase is + // high-entropy and KMS-derived, so an attacker-chosen salt + // buys nothing without it (see security report #552). salt: _, } = kdf else { @@ -2613,7 +3683,7 @@ fn test_decrypt_env_vars_v1_rejects_bad_manifest() { let allowed = test_env_allowed(&["FOO"]); let entry = test_env_entry(&[("FOO", "bar")]); - let ciphertext = test_env_manifest(2, &[entry.clone()]); + let ciphertext = test_env_manifest(2, std::slice::from_ref(&entry)); let err = decrypt_env_vars(&TEST_ENV_CRYPT_KEY, &ciphertext, &allowed).unwrap_err(); assert!(err .to_string() @@ -2631,96 +3701,235 @@ fn test_decrypt_env_vars_v1_rejects_bad_manifest() { assert!(err.to_string().contains("Too many encrypted env entries")); } +#[test] +fn test_validate_luks2_header_rejects_out_of_range_keyslot_area() { + // Redirect the keyslot binary area below the header region. Same length + // so the surrounding header stays intact; "00768" parses to 768, which is + // inside the header copies (< 2 * hdr_size) rather than the metadata gap. + let mut header = include_bytes!("../tests/fixtures/luks_header_good").to_vec(); + let needle = br#""offset":"32768""#; + let replacement = br#""offset":"00768""#; + let mut patched = 0; + let mut i = 0; + while i + needle.len() <= header.len() { + if &header[i..i + needle.len()] == needle { + header[i..i + needle.len()].copy_from_slice(replacement); + patched += 1; + i += needle.len(); + } else { + i += 1; + } + } + assert_eq!(patched, 2, "expected to patch both header copies"); + let error = validate_luks2_headers(&mut &header[..]).unwrap_err(); + assert!(error.to_string().contains("Invalid LUKS keyslot area")); +} + #[cfg(test)] -fn test_app_compose( - manifest_version: serde_json::Value, - os_version: Option<&str>, - platforms: Option<&[&str]>, -) -> AppCompose { +fn test_app_compose(manifest_version: serde_json::Value, platforms: Option<&[&str]>) -> AppCompose { let mut value = serde_json::json!({ "manifest_version": manifest_version, "name": "test", "runner": "docker-compose" }); - if os_version.is_some() || platforms.is_some() { + if platforms.is_some() { value["requirements"] = serde_json::json!({}); } - if let Some(os_version) = os_version { - value["requirements"]["os_version"] = serde_json::json!(os_version); - } if let Some(platforms) = platforms { value["requirements"]["platforms"] = serde_json::json!(platforms); } serde_json::from_value(value).unwrap() } +/// The single hop between `requirements.health_check` and +/// `RegisterCvmRequest.health_check`. Nothing else connects the app's manifest +/// to the gateway's behaviour, so if this is wrong the feature is inert +/// fleet-wide and nothing else fails. +#[test] +fn health_check_opt_in_reaches_registration() { + let opted_in = + compose_with_health_check("docker-compose", serde_json::json!({"health_check": true})); + assert!(health_check_requested(&opted_in)); + + let opted_out = + compose_with_health_check("docker-compose", serde_json::json!({"health_check": false})); + assert!(!health_check_requested(&opted_out)); +} + +/// An app that says nothing must register as "do not poll me", not as an +/// opt-in by omission. +#[test] +fn an_app_that_says_nothing_does_not_ask_to_be_polled() { + let no_requirements = test_app_compose(serde_json::json!("3"), None); + assert!(!health_check_requested(&no_requirements)); + + // Requirements present, but carrying something other than health_check. + let other_requirements = test_app_compose(serde_json::json!("3"), Some(&["dstack-tdx"])); + assert!(!health_check_requested(&other_requirements)); +} + +/// An app that opts into health gating, with `overrides` merged over the +/// `requirements` object so a case can vary one field without restating it. +/// +/// The compose file deliberately declares no `healthcheck:`. Whether the +/// containers can be judged is the runtime's answer, given by the guest agent, +/// not something read out of the YAML here -- so a fixture without one has to +/// pass validation. +#[cfg(test)] +fn compose_with_health_check(runner: &str, overrides: serde_json::Value) -> AppCompose { + let mut requirements = serde_json::json!({"health_check": true}); + for (key, value) in overrides.as_object().expect("an object of overrides") { + requirements[key] = value.clone(); + } + serde_json::from_value(serde_json::json!({ + "manifest_version": "3", + "name": "health-app", + "runner": runner, + "docker_compose_file": "services:\n web:\n image: app:1\n", + "requirements": requirements, + })) + .unwrap() +} + +#[test] +fn health_check_on_a_container_runner_needs_no_health_status_file() { + let compose = compose_with_health_check("docker-compose", serde_json::json!({})); + verify_manifest_feature_requirements(&compose).unwrap(); +} + +/// The `bash` runner starts no containers, so the container fallback has +/// nothing to look at and would answer healthy forever. That is exactly the +/// silent no-op the opt-in exists to avoid. +#[test] +fn health_check_without_containers_to_judge_is_rejected() { + let compose = compose_with_health_check("bash", serde_json::json!({})); + let err = verify_manifest_feature_requirements(&compose).unwrap_err(); + assert!( + err.to_string().contains("needs health_status_file"), + "unexpected error: {err}" + ); +} + +#[test] +fn a_bash_runner_with_a_health_status_file_is_accepted() { + let compose = compose_with_health_check( + "bash", + serde_json::json!({"health_status_file": "/dstack/health"}), + ); + verify_manifest_feature_requirements(&compose).unwrap(); +} + +/// The agent runs in the guest rootfs, so a relative path resolves against +/// whatever its working directory happens to be. +#[test] +fn a_relative_health_status_file_is_rejected() { + let compose = compose_with_health_check( + "docker-compose", + serde_json::json!({"health_status_file": "health"}), + ); + let err = verify_manifest_feature_requirements(&compose).unwrap_err(); + assert!( + err.to_string().contains("absolute path"), + "unexpected error: {err}" + ); +} + +/// Declared but switched off must not be validated as if it were on: turning +/// gating off during an incident should not also have to be a config rewrite. +/// Both of the surviving checks would reject this fixture with the flag on. +#[test] +fn a_disabled_health_check_is_not_validated() { + let compose = compose_with_health_check( + "bash", + serde_json::json!({"health_check": false, "health_status_file": "relative"}), + ); + verify_manifest_feature_requirements(&compose).unwrap(); +} + +/// `requirements` is `deny_unknown_fields`, so a guest image that predates the +/// field refuses the deployment instead of ignoring the request. +#[test] +fn an_unknown_requirements_field_is_refused() { + let parsed = serde_json::from_value::<AppCompose>(serde_json::json!({ + "manifest_version": "3", + "name": "health-app", + "runner": "docker-compose", + "requirements": { "health_check_v2": { "enabled": true } }, + })); + assert!( + parsed.is_err(), + "unknown requirements fields must fail closed" + ); +} + #[test] fn test_manifest_version_policy_rejects_above_guest_max() { - let app_compose = test_app_compose(serde_json::json!("4"), None, None); + let app_compose = test_app_compose(serde_json::json!("4"), None); let err = verify_manifest_version(&app_compose).unwrap_err(); assert!(err.to_string().contains("Unsupported manifest_version")); } #[test] -fn test_os_version_requirement_requires_v3_manifest() { - let app_compose = test_app_compose(serde_json::json!("2"), Some(">=0.6.1"), None); +fn test_requirements_require_v3_manifest() { + let app_compose = test_app_compose(serde_json::json!("2"), Some(&["dstack-tdx"])); let err = verify_manifest_feature_requirements(&app_compose).unwrap_err(); assert!(err.to_string().contains("requires manifest_version")); } #[test] -fn test_os_version_requirement_rejects_too_old_os() { - let app_compose = test_app_compose(serde_json::json!("3"), Some(">=0.6.1"), None); - let err = verify_os_version_requirement(&app_compose, "0.6.0").unwrap_err(); - assert!(err.to_string().contains("Unsupported dstack OS version")); - verify_os_version_requirement(&app_compose, "0.6.1").unwrap(); - verify_os_version_requirement(&app_compose, "0.6.2").unwrap(); +fn test_nerdctl_compose_requires_v3_manifest() { + let mut app_compose = test_app_compose(serde_json::json!(2), None); + app_compose.runner = "nerdctl-compose".to_string(); + let err = verify_manifest_feature_requirements(&app_compose).unwrap_err(); + assert!(err.to_string().contains("nerdctl-compose requires")); + + app_compose.manifest_version = "3".to_string(); + verify_manifest_feature_requirements(&app_compose).unwrap(); } #[test] -fn test_os_version_requirement_accepts_semver_requirement_ranges() { - let app_compose = test_app_compose(serde_json::json!("3"), Some(">=0.6.0, <0.7.0"), None); - verify_os_version_requirement(&app_compose, "0.6.0").unwrap(); - verify_os_version_requirement(&app_compose, "0.6.9").unwrap(); - let err = verify_os_version_requirement(&app_compose, "0.7.0").unwrap_err(); - assert!(err.to_string().contains("Unsupported dstack OS version")); +fn test_multiple_init_scripts_require_v3_manifest() { + let mut app_compose = test_app_compose(serde_json::json!(2), None); + app_compose.init_script = vec!["echo one".into(), "echo two".into()]; + let err = verify_manifest_feature_requirements(&app_compose).unwrap_err(); + assert!(err + .to_string() + .contains("multiple init scripts require manifest_version")); + + app_compose.manifest_version = "3".into(); + verify_manifest_feature_requirements(&app_compose).unwrap(); } #[test] -fn test_os_version_requirement_rejects_invalid_semver_strings() { - let app_compose = test_app_compose(serde_json::json!("3"), Some(">=0.6.0.a0"), None); - let err = verify_os_version_requirement(&app_compose, "0.6.0").unwrap_err(); - assert!(err.to_string().contains("Invalid requirements.os_version")); - - let app_compose = test_app_compose(serde_json::json!("3"), Some(">=0.6.0-a0"), None); - let err = verify_os_version_requirement(&app_compose, "0.6.0.a0").unwrap_err(); +fn test_snapshotter_is_rejected_for_other_runners() { + let mut app_compose = test_app_compose(serde_json::json!("3"), None); + app_compose.snapshotter = Some(dstack_types::ContainerSnapshotter::Stargz); + let err = verify_manifest_feature_requirements(&app_compose).unwrap_err(); assert!(err .to_string() - .contains("Invalid current dstack OS version")); + .contains("snapshotter is only supported by the nerdctl-compose runner")); } #[test] fn test_platform_requirements_accept_matching_platform() { let app_compose = test_app_compose( serde_json::json!("3"), - None, Some(&["dstack-gcp-tdx", "dstack-tdx"]), ); - verify_platform_requirements(&app_compose, AttestationMode::DstackGcpTdx).unwrap(); - verify_platform_requirements(&app_compose, AttestationMode::DstackTdx).unwrap(); + verify_platform_requirements(&app_compose, TeeVariant::DstackGcpTdx).unwrap(); + verify_platform_requirements(&app_compose, TeeVariant::DstackTdx).unwrap(); } #[test] fn test_platform_requirements_reject_non_matching_platform() { - let app_compose = test_app_compose(serde_json::json!("3"), None, Some(&["dstack-gcp-tdx"])); - let err = - verify_platform_requirements(&app_compose, AttestationMode::DstackAmdSevSnp).unwrap_err(); + let app_compose = test_app_compose(serde_json::json!("3"), Some(&["dstack-gcp-tdx"])); + let err = verify_platform_requirements(&app_compose, TeeVariant::DstackAmdSevSnp).unwrap_err(); assert!(err.to_string().contains("Unsupported attestation platform")); } #[test] fn test_platform_requirements_require_v3_manifest() { - let app_compose = test_app_compose(serde_json::json!("2"), None, Some(&["dstack-gcp-tdx"])); + let app_compose = test_app_compose(serde_json::json!("2"), Some(&["dstack-gcp-tdx"])); let err = verify_manifest_feature_requirements(&app_compose).unwrap_err(); assert!(err.to_string().contains("requires manifest_version")); } @@ -2740,25 +3949,25 @@ fn test_empty_requirements_require_v3_manifest() { #[test] fn test_platform_requirements_omitted_accepts_any_platform() { - let app_compose = test_app_compose(serde_json::json!("3"), None, None); - verify_platform_requirements(&app_compose, AttestationMode::DstackAmdSevSnp).unwrap(); + let app_compose = test_app_compose(serde_json::json!("3"), None); + verify_platform_requirements(&app_compose, TeeVariant::DstackAmdSevSnp).unwrap(); } #[test] fn test_platform_requirements_explicit_empty_rejects_all_platforms() { - let app_compose = test_app_compose(serde_json::json!("3"), None, Some(&[])); - let err = verify_platform_requirements(&app_compose, AttestationMode::DstackTdx).unwrap_err(); + let app_compose = test_app_compose(serde_json::json!("3"), Some(&[])); + let err = verify_platform_requirements(&app_compose, TeeVariant::DstackTdx).unwrap_err(); assert!(err.to_string().contains("Unsupported attestation platform")); - let app_compose = test_app_compose(serde_json::json!("2"), None, Some(&[])); + let app_compose = test_app_compose(serde_json::json!("2"), Some(&[])); let err = verify_manifest_feature_requirements(&app_compose).unwrap_err(); assert!(err.to_string().contains("requires manifest_version")); } #[test] fn test_platform_requirements_reject_invalid_platform_value() { - let app_compose = test_app_compose(serde_json::json!("3"), None, Some(&["gcptdx"])); - let err = verify_platform_requirements(&app_compose, AttestationMode::DstackTdx).unwrap_err(); + let app_compose = test_app_compose(serde_json::json!("3"), Some(&["gcptdx"])); + let err = verify_platform_requirements(&app_compose, TeeVariant::DstackTdx).unwrap_err(); assert!(err .to_string() .contains("Invalid requirements.platforms[0]")); @@ -2775,12 +3984,12 @@ fn test_tdx_measure_acpi_tables_requirement_matches_vm_config() { } })) .unwrap(); - verify_tdx_measure_acpi_tables_requirement(&app_compose, r#"{}"#, AttestationMode::DstackTdx) + verify_tdx_measure_acpi_tables_requirement(&app_compose, r#"{}"#, TeeVariant::DstackTdx) .unwrap(); let err = verify_tdx_measure_acpi_tables_requirement( &app_compose, r#"{"tdx_attestation_variant":"lite"}"#, - AttestationMode::DstackTdx, + TeeVariant::DstackTdx, ) .unwrap_err(); assert!(err.to_string().contains("tdx_measure_acpi_tables=true")); @@ -2797,15 +4006,12 @@ fn test_tdx_measure_acpi_tables_requirement_matches_vm_config() { verify_tdx_measure_acpi_tables_requirement( &app_compose, r#"{"tdx_attestation_variant":"lite"}"#, - AttestationMode::DstackTdx, + TeeVariant::DstackTdx, ) .unwrap(); - let err = verify_tdx_measure_acpi_tables_requirement( - &app_compose, - r#"{}"#, - AttestationMode::DstackTdx, - ) - .unwrap_err(); + let err = + verify_tdx_measure_acpi_tables_requirement(&app_compose, r#"{}"#, TeeVariant::DstackTdx) + .unwrap_err(); assert!(err.to_string().contains("tdx_measure_acpi_tables=false")); } @@ -2823,7 +4029,7 @@ fn test_tdx_measure_acpi_tables_requirement_ignored_on_non_tdx() { verify_tdx_measure_acpi_tables_requirement( &app_compose, r#"{"tdx_attestation_variant":"lite"}"#, - AttestationMode::DstackAmdSevSnp, + TeeVariant::DstackAmdSevSnp, ) .unwrap(); } @@ -2895,30 +4101,389 @@ fn test_launch_token_from_user_config_rejects_missing_or_invalid_token() { .contains("failed to parse user_config as JSON")); } -#[test] -fn test_os_release_value_parses_quoted_version_id() { - let content = r#" -NAME="DStack" -VERSION_ID="0.6.1" -"#; - assert_eq!( - os_release_value(content, "VERSION_ID").as_deref(), - Some("0.6.1") - ); +#[cfg(test)] +mod kms_provider_inventory_tests { + use super::{kms_rpc_url, validate_key_provider_inputs}; + use dstack_types::KeyProviderKind; + + #[test] + fn normalizes_kms_rpc_urls_once() { + assert_eq!(kms_rpc_url("https://kms.test"), "https://kms.test/prpc"); + assert_eq!(kms_rpc_url("https://kms.test/"), "https://kms.test/prpc"); + assert_eq!( + kms_rpc_url("https://kms.test/prpc"), + "https://kms.test/prpc" + ); + assert_eq!( + kms_rpc_url("https://kms.test/prpc/"), + "https://kms.test/prpc" + ); + } + + #[test] + fn local_key_providers_do_not_require_kms_inventory() { + let no_urls = Vec::new(); + assert!(validate_key_provider_inputs(KeyProviderKind::Local, &no_urls).is_ok()); + assert!(validate_key_provider_inputs(KeyProviderKind::Tpm, &no_urls).is_ok()); + assert!(validate_key_provider_inputs(KeyProviderKind::None, &no_urls).is_ok()); + let error = validate_key_provider_inputs(KeyProviderKind::Kms, &no_urls).unwrap_err(); + assert!(error.to_string().contains("No KMS URLs are set")); + } } -#[test] -fn test_unquote_os_release_value_handles_quoting_styles() { - assert_eq!(unquote_os_release_value("0.6.1"), "0.6.1"); - assert_eq!(unquote_os_release_value("\"0.6.1\""), "0.6.1"); - assert_eq!(unquote_os_release_value("'0.6.1'"), "0.6.1"); - // Double-quoted: backslash escapes the next character. - assert_eq!(unquote_os_release_value(r#""a\"b""#), "a\"b"); - assert_eq!(unquote_os_release_value(r#""a\\b""#), r"a\b"); - assert_eq!(unquote_os_release_value(r#""a\\\"b""#), r#"a\"b"#); - // Single-quoted: no escape sequences. - assert_eq!(unquote_os_release_value(r"'a\\b'"), r"a\\b"); - // Unbalanced/degenerate quotes are returned verbatim. - assert_eq!(unquote_os_release_value("\""), "\""); - assert_eq!(unquote_os_release_value("\"a"), "\"a"); +#[cfg(test)] +mod gateway_registration_refresh_tests { + use super::{ + carry_cluster_preference, gateway_rpc_url, order_by_stickiness, record_accepting_url, + wireguard_endpoint_hosts, GatewayKeyStore, IssuedClientCerts, + }; + use std::os::unix::fs::PermissionsExt as _; + + fn key_store(cert_not_after: u64) -> GatewayKeyStore { + GatewayKeyStore { + client_cert: "sentinel-client-cert".into(), + client_cert_with_quote: "sentinel-quoted-cert".into(), + client_key: "sentinel-client-key".into(), + cert_not_after, + wg_sk: "sentinel-wg-private".into(), + wg_pk: "sentinel-wg-public".into(), + last_url: None, + } + } + + fn urls(list: &[&str]) -> Vec<String> { + list.iter().map(|url| url.to_string()).collect() + } + + fn ordered(list: &[String], last: Option<&str>) -> Vec<String> { + order_by_stickiness(list, last) + .into_iter() + .cloned() + .collect() + } + + /// With nothing remembered the configured order stands, which is the + /// behaviour every existing deployment already has. + #[test] + fn without_a_remembered_url_the_configured_order_stands() { + let list = urls(&["https://a", "https://b", "https://c"]); + assert_eq!(ordered(&list, None), list); + } + + /// The point: a CVM that moved to the second gateway during an outage stays + /// there instead of snapping back the moment the first recovers. Each move + /// rewrites the instance record from a different node's memory. + #[test] + fn the_last_accepting_url_is_tried_first() { + let list = urls(&["https://a", "https://b", "https://c"]); + assert_eq!( + ordered(&list, Some("https://b")), + urls(&["https://b", "https://a", "https://c"]) + ); + } + + /// A preference, not a pin: if the remembered one is down it costs one + /// failed attempt and the rest follow in configured order. + #[test] + fn the_remaining_urls_keep_their_configured_order() { + let list = urls(&["https://a", "https://b", "https://c", "https://d"]); + assert_eq!( + ordered(&list, Some("https://c")), + urls(&["https://c", "https://a", "https://b", "https://d"]) + ); + } + + /// An operator removing a URL from the config must not resurrect it. + #[test] + fn a_remembered_url_no_longer_configured_is_ignored() { + let list = urls(&["https://a", "https://b"]); + assert_eq!(ordered(&list, Some("https://gone")), list); + } + + #[test] + fn every_url_is_tried_exactly_once() { + let list = urls(&["https://a", "https://b", "https://c"]); + for last in [ + None, + Some("https://a"), + Some("https://b"), + Some("https://c"), + ] { + let mut seen = ordered(&list, last); + assert_eq!(seen.len(), list.len(), "last={last:?}"); + seen.sort(); + let mut expected = list.clone(); + expected.sort(); + assert_eq!(seen, expected, "last={last:?}"); + } + } + + /// The cache is what carries the preference across a refresh, so it has to + /// survive the round trip -- and an older cache without the field must + /// still load. + #[test] + fn the_remembered_url_round_trips_through_the_cache() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("gateway-cache.json"); + let mut store = key_store(u64::MAX); + store.last_url = Some("https://b".into()); + store.save_to(&path).expect("save"); + let loaded = GatewayKeyStore::load_from(&path).expect("load"); + assert_eq!(loaded.last_url.as_deref(), Some("https://b")); + + let legacy = serde_json::to_value(key_store(u64::MAX)) + .map(|mut value| { + value.as_object_mut().unwrap().remove("last_url"); + value + }) + .expect("serialize"); + std::fs::write(&path, legacy.to_string()).expect("write"); + let loaded = + GatewayKeyStore::load_from(&path).expect("a cache without the field must load"); + assert_eq!(loaded.last_url, None); + } + + fn issued_certs(cert_not_after: u64) -> IssuedClientCerts { + IssuedClientCerts { + client_cert: "renewed-cert".into(), + client_cert_with_quote: "renewed-cert-with-quote".into(), + client_key: "renewed-key".into(), + cert_not_after, + } + } + + /// A certificate renewal rebuilds the store from scratch. It must carry + /// the WireGuard identity — the gateway maps this peer by its public + /// key — and the registration preference, or every renewal would drift + /// the CVM back onto the first configured URL. + #[test] + fn a_certificate_renewal_carries_the_wireguard_identity_and_the_preference() { + let mut cached = key_store(0); + cached.last_url = Some("https://b".into()); + let renewed = GatewayKeyStore::renewed( + Some(cached), + || panic!("a cached WireGuard identity must be reused, not regenerated"), + issued_certs(u64::MAX), + ) + .expect("renew"); + assert_eq!(renewed.wg_sk, "sentinel-wg-private"); + assert_eq!(renewed.wg_pk, "sentinel-wg-public"); + assert_eq!(renewed.last_url.as_deref(), Some("https://b")); + assert_eq!(renewed.client_cert, "renewed-cert"); + assert_eq!(renewed.cert_not_after, u64::MAX); + } + + /// A cold start has nothing to carry: fresh WireGuard identity, no + /// preference. + #[test] + fn a_cold_start_generates_a_fresh_wireguard_identity_and_no_preference() { + let renewed = GatewayKeyStore::renewed( + None, + || Ok(("fresh-wg-private".into(), "fresh-wg-public".into())), + issued_certs(u64::MAX), + ) + .expect("renew"); + assert_eq!(renewed.wg_sk, "fresh-wg-private"); + assert_eq!(renewed.wg_pk, "fresh-wg-public"); + assert_eq!(renewed.last_url, None); + } + + /// The sequence a refresh tick runs for the default cluster when the + /// certificate expires: rebuild the store, save it over the cache, then + /// read the preference back for the registration round. The rebuilt store + /// is what gets written, so it must already carry the preference — this + /// is exactly the sequence that used to lose it. + #[test] + fn the_preference_survives_the_renewal_that_rewrites_the_cache() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("gateway-cache.json"); + let mut cached = key_store(0); // expired certificate forces the rebuild + cached.last_url = Some("https://b".into()); + cached.save_to(&path).expect("save"); + + let renewed = GatewayKeyStore::renewed( + GatewayKeyStore::load_from(&path), + || panic!("a cached WireGuard identity must be reused, not regenerated"), + issued_certs(u64::MAX), + ) + .expect("renew"); + renewed.save_to(&path).expect("save renewed"); + + let mut store = renewed; + carry_cluster_preference(&mut store, &path, &urls(&["https://a", "https://b"])); + assert_eq!(store.last_url.as_deref(), Some("https://b")); + } + + /// An additional cluster's store is cloned from the primary's; the + /// preference that counts is the one in the cluster's own cache. + #[test] + fn an_additional_cluster_does_not_inherit_the_primary_preference() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("gateway-cache-second.json"); + let mut cluster_cache = key_store(u64::MAX); + cluster_cache.last_url = Some("https://b".into()); + cluster_cache.save_to(&path).expect("save"); + + let mut store = key_store(u64::MAX); + store.last_url = Some("https://primary".into()); + carry_cluster_preference(&mut store, &path, &urls(&["https://a", "https://b"])); + assert_eq!(store.last_url.as_deref(), Some("https://b")); + } + + /// No cache on disk means no preference, whatever the resolved store + /// happened to carry. + #[test] + fn a_missing_cluster_cache_clears_the_carried_preference() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("absent.json"); + let mut store = key_store(u64::MAX); + store.last_url = Some("https://primary".into()); + carry_cluster_preference(&mut store, &path, &urls(&["https://primary"])); + assert_eq!(store.last_url, None); + } + + /// An operator removing a URL from the config must not see the cache + /// resurrect it. + #[test] + fn the_carry_drops_a_url_removed_from_the_config() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("gateway-cache.json"); + let mut cached = key_store(u64::MAX); + cached.last_url = Some("https://gone".into()); + cached.save_to(&path).expect("save"); + + let mut store = key_store(u64::MAX); + carry_cluster_preference(&mut store, &path, &urls(&["https://a"])); + assert_eq!(store.last_url, None); + } + + /// Only an acceptance updates the preference: a round where every URL + /// failed says nothing about where the instance record lives, and + /// clearing on it would erase the fleet's spread in one bad tick. + #[test] + fn a_round_where_every_url_failed_keeps_the_preference() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("gateway-cache.json"); + let mut store = key_store(u64::MAX); + store.last_url = Some("https://b".into()); + store.save_to(&path).expect("save"); + + record_accepting_url("default", &mut store, &path, None); + assert_eq!(store.last_url.as_deref(), Some("https://b")); + let on_disk = GatewayKeyStore::load_from(&path).expect("load"); + assert_eq!(on_disk.last_url.as_deref(), Some("https://b")); + } + + /// An acceptance by a different node is recorded in memory and on disk. + #[test] + fn an_acceptance_moves_and_persists_the_preference() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("gateway-cache.json"); + let mut store = key_store(u64::MAX); + store.last_url = Some("https://a".into()); + + record_accepting_url("default", &mut store, &path, Some("https://c".into())); + assert_eq!(store.last_url.as_deref(), Some("https://c")); + let on_disk = GatewayKeyStore::load_from(&path).expect("load"); + assert_eq!(on_disk.last_url.as_deref(), Some("https://c")); + } + + /// The first acceptance of a boot is recorded too — that is what the + /// next refresh's ordering is built from. + #[test] + fn the_first_acceptance_is_recorded() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("gateway-cache.json"); + let mut store = key_store(u64::MAX); + + record_accepting_url("default", &mut store, &path, Some("https://a".into())); + assert_eq!(store.last_url.as_deref(), Some("https://a")); + let on_disk = GatewayKeyStore::load_from(&path).expect("load"); + assert_eq!(on_disk.last_url.as_deref(), Some("https://a")); + } + + /// Re-acceptance by the remembered node changes nothing, so nothing is + /// written. + #[test] + fn re_acceptance_by_the_same_url_does_not_rewrite_the_cache() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("gateway-cache.json"); + let mut store = key_store(u64::MAX); + store.last_url = Some("https://c".into()); + + record_accepting_url("default", &mut store, &path, Some("https://c".into())); + assert_eq!(store.last_url.as_deref(), Some("https://c")); + assert!( + !path.exists(), + "an unchanged preference must not be written" + ); + } + + #[test] + fn gateway_rpc_urls_are_normalized_once() { + assert_eq!( + gateway_rpc_url("https://gateway.test"), + "https://gateway.test/prpc" + ); + assert_eq!( + gateway_rpc_url("https://gateway.test/"), + "https://gateway.test/prpc" + ); + assert_eq!( + gateway_rpc_url("https://gateway.test/prpc"), + "https://gateway.test/prpc" + ); + assert_eq!( + gateway_rpc_url("https://gateway.test/prpc/"), + "https://gateway.test/prpc" + ); + } + + #[test] + fn key_store_round_trip_is_private_and_stable() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("gateway-cache.json"); + let original = key_store(10_000); + original.save_to(&path).unwrap(); + assert_eq!(path.metadata().unwrap().permissions().mode() & 0o777, 0o600); + let loaded = GatewayKeyStore::load_from(&path).unwrap(); + assert_eq!(loaded.wg_sk, original.wg_sk); + assert_eq!(loaded.wg_pk, original.wg_pk); + assert_eq!(loaded.client_key, original.client_key); + } + + #[test] + fn malformed_replacement_does_not_overwrite_working_cache() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("gateway-cache.json"); + let original = key_store(10_000); + original.save_to(&path).unwrap(); + let before = std::fs::read(&path).unwrap(); + let invalid_target = directory.path().join("missing-parent/cache.json"); + assert!(key_store(20_000).save_to(&invalid_target).is_ok()); + assert_eq!(std::fs::read(&path).unwrap(), before); + std::fs::write(&path, b"not-json").unwrap(); + assert!(GatewayKeyStore::load_from(&path).is_none()); + } + + #[test] + fn certificate_refresh_boundary_is_strict_and_overflow_safe() { + assert!(key_store(1_601).is_cert_valid_at(1_000)); + assert!(!key_store(1_600).is_cert_valid_at(1_000)); + assert!(!key_store(u64::MAX).is_cert_valid_at(u64::MAX)); + } + + #[test] + fn wireguard_endpoint_hosts_support_dns_ipv4_and_ipv6() { + let config = r#" +Endpoint = gateway.example.com:51820 +Endpoint = 192.0.2.1:51821 +Endpoint = [2001:db8::1]:51822 +"#; + assert_eq!( + wireguard_endpoint_hosts(config).unwrap(), + ["gateway.example.com", "192.0.2.1", "2001:db8::1"] + ); + assert!(wireguard_endpoint_hosts("Endpoint = missing-port").is_err()); + } } diff --git a/dstack/dstack-util/src/system_setup/config_id_verifier.rs b/dstack/dstack-util/src/system_setup/config_id_verifier.rs index c350b32e6..3e71cd76c 100644 --- a/dstack/dstack-util/src/system_setup/config_id_verifier.rs +++ b/dstack/dstack-util/src/system_setup/config_id_verifier.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::{bail, Context, Result}; -use dstack_attest::attestation::{Attestation, AttestationMode, AttestationQuote}; +use dstack_attest::attestation::{detect_tee_variant, Attestation, AttestationQuote, TeeVariant}; use dstack_types::{ mr_config::{MrConfig, MrConfigV3}, shared_filenames::{host_shared_dir, SYS_CONFIG}, @@ -12,8 +12,10 @@ use dstack_types::{ use tracing::info; #[derive(Clone, Copy)] -struct ExpectedMrConfig<'a> { +struct LocalMrConfigValues<'a> { compose_hash: &'a [u8; 32], + gpu_policy_hash: &'a [u8; 32], + init_script_hashes: &'a [Vec<u8>], app_id: &'a [u8; 20], instance_id: &'a [u8], key_provider: KeyProviderKind, @@ -51,7 +53,7 @@ fn read_snp_host_data() -> Result<[u8; 32]> { Ok(parsed.host_data) } -/// Verify the mr_config_id matches the expected value +/// Verify the mr_config_id matches values observed locally by the guest. /// /// Configuration ID format /// The mr_config_id is a 48 bytes value in the following format: @@ -65,37 +67,41 @@ fn read_snp_host_data() -> Result<[u8; 32]> { /// - key_provider_id: [u8] // KMS CA pubkey, local-sgx MR, or empty for none/tpm pub fn verify_mr_config_id( compose_hash: &[u8; 32], + gpu_policy_hash: &[u8; 32], + init_script_hashes: &[Vec<u8>], app_id: &[u8; 20], instance_id: &[u8], key_provider: KeyProviderKind, key_provider_id: &[u8], ) -> Result<()> { - let mode = AttestationMode::detect().context("Failed to detect attestation mode")?; - let expected = ExpectedMrConfig { + let mode = detect_tee_variant().context("Failed to detect attestation mode")?; + let local = LocalMrConfigValues { compose_hash, + gpu_policy_hash, + init_script_hashes, app_id, instance_id, key_provider, key_provider_id, }; - verify_mr_config_id_for_mode(mode, expected) + verify_mr_config_id_for_mode(mode, local) } -fn verify_mr_config_id_for_mode( - mode: AttestationMode, - expected: ExpectedMrConfig<'_>, -) -> Result<()> { +fn verify_mr_config_id_for_mode(mode: TeeVariant, local: LocalMrConfigValues<'_>) -> Result<()> { match mode { - AttestationMode::DstackAmdSevSnp => verify_snp_mr_config(expected), + TeeVariant::DstackAmdSevSnp => verify_snp_mr_config(local), // AWS PCR8 is computed by the guest from measured reality (MrConfig V2 // in measure_app_info); there is no host-supplied claim to cross-check. // The key_provider_id pin is enforced by verify_key_provider_id. - AttestationMode::DstackAwsNitroTpm => Ok(()), - _ => verify_tdx_mr_config_id(expected), + TeeVariant::DstackAwsNitroTpm => Ok(()), + // Nitro Enclave binds the image through the signed NSM document and + // the app ID through its runtime event. It has no TDX mr_config_id. + TeeVariant::DstackNitroEnclave => Ok(()), + _ => verify_tdx_mr_config_id(local), } } -fn verify_tdx_mr_config_id(expected: ExpectedMrConfig<'_>) -> Result<()> { +fn verify_tdx_mr_config_id(local: LocalMrConfigValues<'_>) -> Result<()> { let read_mr_config_id = read_mr_config_id().context("Failed to read mr_config_id")?; info!("mr_config_id: {}", hex::encode(read_mr_config_id)); let mr_config_document = if read_mr_config_id[0] == 3 { @@ -103,33 +109,33 @@ fn verify_tdx_mr_config_id(expected: ExpectedMrConfig<'_>) -> Result<()> { } else { None }; - verify_tdx_mr_config_id_value(read_mr_config_id, mr_config_document.as_deref(), expected) + verify_tdx_mr_config_id_value(read_mr_config_id, mr_config_document.as_deref(), local) } fn verify_tdx_mr_config_id_value( read_mr_config_id: [u8; 48], mr_config_document: Option<&str>, - expected: ExpectedMrConfig<'_>, + local: LocalMrConfigValues<'_>, ) -> Result<()> { if read_mr_config_id == [0u8; 48] { return Ok(()); } let expected_mr_config_id = match read_mr_config_id[0] { 1 => MrConfig::V1 { - compose_hash: expected.compose_hash, + compose_hash: local.compose_hash, } .to_mr_config_id(), 2 => MrConfig::V2 { - compose_hash: expected.compose_hash, - app_id: expected.app_id, - key_provider: expected.key_provider, - key_provider_id: expected.key_provider_id, + compose_hash: local.compose_hash, + app_id: local.app_id, + key_provider: local.key_provider, + key_provider_id: local.key_provider_id, } .to_mr_config_id(), 3 => { let mr_config_document = mr_config_document.context("mr_config is required for TDX MR_CONFIG_ID v3")?; - verify_mr_config_v3_document(mr_config_document, expected)?; + verify_mr_config_v3_document(mr_config_document, local)?; MrConfigV3::tdx_mr_config_id_from_document(mr_config_document) } _ => bail!("Invalid mr_config_id version"), @@ -140,9 +146,9 @@ fn verify_tdx_mr_config_id_value( Ok(()) } -fn verify_snp_mr_config(expected: ExpectedMrConfig<'_>) -> Result<()> { +fn verify_snp_mr_config(local: LocalMrConfigValues<'_>) -> Result<()> { let mr_config_document = read_mr_config_document().context("Failed to read SNP mr_config")?; - verify_mr_config_v3_document(&mr_config_document, expected)?; + verify_mr_config_v3_document(&mr_config_document, local)?; let read_host_data = read_snp_host_data().context("Failed to read SNP HOST_DATA")?; info!("snp host_data: {}", hex::encode(read_host_data)); if MrConfigV3::snp_host_data_from_document(&mr_config_document) != read_host_data { @@ -153,27 +159,43 @@ fn verify_snp_mr_config(expected: ExpectedMrConfig<'_>) -> Result<()> { fn verify_mr_config_v3_document( mr_config_document: &str, - expected: ExpectedMrConfig<'_>, + local: LocalMrConfigValues<'_>, ) -> Result<MrConfigV3> { let mr_config = MrConfigV3::from_document(mr_config_document).context("Invalid mr_config document")?; if mr_config.version != 3 { bail!("mr_config version must be 3"); } - if mr_config.compose_hash.as_slice() != expected.compose_hash { + if mr_config.compose_hash.as_slice() != local.compose_hash { bail!("Invalid mr_config compose_hash"); } - if mr_config.app_id.as_slice() != expected.app_id { - bail!("Invalid mr_config app_id"); + if let Some(declared_gpu_policy_hash) = mr_config.gpu_policy_hash.as_deref() { + if declared_gpu_policy_hash != local.gpu_policy_hash { + bail!("Invalid mr_config gpu_policy_hash"); + } + } + if let Some(init_script_hashes) = mr_config.init_script_hashes.as_deref() { + if init_script_hashes != local.init_script_hashes { + bail!("Invalid mr_config init_script_hashes"); + } + } + if let Some(app_id) = mr_config.app_id.as_deref() { + if app_id != local.app_id { + bail!("Invalid mr_config app_id"); + } } - if mr_config.instance_id.as_slice() != expected.instance_id { - bail!("Invalid mr_config instance_id"); + if let Some(instance_id) = mr_config.instance_id.as_deref() { + if instance_id != local.instance_id { + bail!("Invalid mr_config instance_id"); + } } - if mr_config.key_provider != expected.key_provider { + if mr_config.key_provider != local.key_provider { bail!("Invalid mr_config key_provider"); } - if mr_config.key_provider_id.as_slice() != expected.key_provider_id { - bail!("Invalid mr_config key_provider_id"); + if let Some(key_provider_id) = mr_config.key_provider_id.as_deref() { + if key_provider_id != local.key_provider_id { + bail!("Invalid mr_config key_provider_id"); + } } Ok(mr_config) } @@ -194,54 +216,248 @@ mod tests { #[test] fn tdx_mr_config_id_v3_accepts_document_value() -> Result<()> { let compose_hash = [0x22u8; 32]; + let gpu_policy_hash = [0x55u8; 32]; let app_id = [0x11u8; 20]; let instance_id = [0x44u8; 20]; let key_provider_id = [0x33u8; 32]; let mr_config = MrConfigV3::new( app_id.to_vec(), compose_hash.to_vec(), + Some(gpu_policy_hash.to_vec()), KeyProviderKind::Kms, key_provider_id.to_vec(), instance_id.to_vec(), ); let document = mr_config.to_canonical_json(); - let expected = ExpectedMrConfig { + let local = LocalMrConfigValues { compose_hash: &compose_hash, + gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &[], app_id: &app_id, instance_id: &instance_id, key_provider: KeyProviderKind::Kms, key_provider_id: &key_provider_id, }; - verify_tdx_mr_config_id_value(mr_config.to_tdx_mr_config_id(), Some(&document), expected) + verify_tdx_mr_config_id_value(mr_config.to_tdx_mr_config_id(), Some(&document), local) } #[test] fn mr_config_v3_document_must_match_expected_app_info() { let compose_hash = [0x22u8; 32]; + let gpu_policy_hash = [0x55u8; 32]; let app_id = [0x11u8; 20]; let instance_id = [0x44u8; 20]; let key_provider_id = [0x33u8; 32]; let document = MrConfigV3::new( app_id.to_vec(), compose_hash.to_vec(), + Some(gpu_policy_hash.to_vec()), KeyProviderKind::Kms, key_provider_id.to_vec(), instance_id.to_vec(), ) .to_canonical_json(); let wrong_app_id = [0x12u8; 20]; - let expected = ExpectedMrConfig { + let local = LocalMrConfigValues { compose_hash: &compose_hash, + gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &[], app_id: &wrong_app_id, instance_id: &instance_id, key_provider: KeyProviderKind::Kms, key_provider_id: &key_provider_id, }; - match verify_mr_config_v3_document(&document, expected) { + match verify_mr_config_v3_document(&document, local) { Ok(_) => panic!("mismatched app_id must reject"), Err(err) => assert!(err.to_string().contains("Invalid mr_config app_id")), } } + + #[test] + fn mr_config_v3_skips_app_id_check_when_field_is_missing() -> Result<()> { + let compose_hash = [0x22u8; 32]; + let gpu_policy_hash = [0x55u8; 32]; + let app_id = [0x11u8; 20]; + let instance_id = [0x44u8; 20]; + let key_provider_id = [0x33u8; 32]; + let document = MrConfigV3::new( + Vec::new(), + compose_hash.to_vec(), + Some(gpu_policy_hash.to_vec()), + KeyProviderKind::Kms, + key_provider_id.to_vec(), + instance_id.to_vec(), + ) + .to_canonical_json(); + let local = LocalMrConfigValues { + compose_hash: &compose_hash, + gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &[], + app_id: &app_id, + instance_id: &instance_id, + key_provider: KeyProviderKind::Kms, + key_provider_id: &key_provider_id, + }; + + verify_mr_config_v3_document(&document, local)?; + Ok(()) + } + + #[test] + fn mr_config_v3_document_must_match_expected_gpu_policy_hash() { + let compose_hash = [0x22u8; 32]; + let gpu_policy_hash = [0x55u8; 32]; + let app_id = [0x11u8; 20]; + let instance_id = [0x44u8; 20]; + let key_provider_id = [0x33u8; 32]; + let document = MrConfigV3::new( + app_id.to_vec(), + compose_hash.to_vec(), + Some(gpu_policy_hash.to_vec()), + KeyProviderKind::Kms, + key_provider_id.to_vec(), + instance_id.to_vec(), + ) + .to_canonical_json(); + let wrong_gpu_policy_hash = [0x56u8; 32]; + let local = LocalMrConfigValues { + compose_hash: &compose_hash, + gpu_policy_hash: &wrong_gpu_policy_hash, + init_script_hashes: &[], + app_id: &app_id, + instance_id: &instance_id, + key_provider: KeyProviderKind::Kms, + key_provider_id: &key_provider_id, + }; + + match verify_mr_config_v3_document(&document, local) { + Ok(_) => panic!("mismatched gpu_policy_hash must reject"), + Err(err) => assert!(err + .to_string() + .contains("Invalid mr_config gpu_policy_hash")), + } + } + + #[test] + fn mr_config_v3_skips_gpu_policy_hash_check_when_field_is_missing() -> Result<()> { + let compose_hash = [0x22u8; 32]; + let actual_gpu_policy_hash = [0x55u8; 32]; + let app_id = [0x11u8; 20]; + let instance_id = [0x44u8; 20]; + let key_provider_id = [0x33u8; 32]; + let mr_config = MrConfigV3::new( + app_id.to_vec(), + compose_hash.to_vec(), + None, + KeyProviderKind::Kms, + key_provider_id.to_vec(), + instance_id.to_vec(), + ); + let document = mr_config.to_canonical_json(); + let local = LocalMrConfigValues { + compose_hash: &compose_hash, + gpu_policy_hash: &actual_gpu_policy_hash, + init_script_hashes: &[], + app_id: &app_id, + instance_id: &instance_id, + key_provider: KeyProviderKind::Kms, + key_provider_id: &key_provider_id, + }; + + verify_tdx_mr_config_id_value(mr_config.to_tdx_mr_config_id(), Some(&document), local) + } + + #[test] + fn mr_config_v3_document_rejects_mismatched_init_script_hashes() { + let compose_hash = [0x22u8; 32]; + let gpu_policy_hash = [0x55u8; 32]; + let app_id = [0x11u8; 20]; + let instance_id = [0x44u8; 20]; + let declared_hashes = vec![vec![0xaau8; 32]]; + let actual_hashes = vec![vec![0xbbu8; 32]]; + let document = MrConfigV3::new( + app_id.to_vec(), + compose_hash.to_vec(), + None, + KeyProviderKind::None, + Vec::new(), + instance_id.to_vec(), + ) + .with_init_script_hashes(declared_hashes) + .to_canonical_json(); + let local = LocalMrConfigValues { + compose_hash: &compose_hash, + gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &actual_hashes, + app_id: &app_id, + instance_id: &instance_id, + key_provider: KeyProviderKind::None, + key_provider_id: &[], + }; + + assert!(verify_mr_config_v3_document(&document, local) + .unwrap_err() + .to_string() + .contains("Invalid mr_config init_script_hashes")); + } + + #[test] + fn mr_config_v3_document_skips_init_script_check_when_field_is_missing() { + let compose_hash = [0x22u8; 32]; + let gpu_policy_hash = [0x55u8; 32]; + let app_id = [0x11u8; 20]; + let instance_id = [0x44u8; 20]; + let mut document = serde_json::to_value(MrConfigV3::new( + app_id.to_vec(), + compose_hash.to_vec(), + None, + KeyProviderKind::None, + Vec::new(), + instance_id.to_vec(), + )) + .unwrap(); + document + .as_object_mut() + .unwrap() + .remove("init_script_hashes"); + let local_without_scripts = LocalMrConfigValues { + compose_hash: &compose_hash, + gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &[], + app_id: &app_id, + instance_id: &instance_id, + key_provider: KeyProviderKind::None, + key_provider_id: &[], + }; + + verify_mr_config_v3_document(&document.to_string(), local_without_scripts).unwrap(); + + let actual_hashes = vec![vec![0xaau8; 32]]; + let local_with_script = LocalMrConfigValues { + init_script_hashes: &actual_hashes, + ..local_without_scripts + }; + verify_mr_config_v3_document(&document.to_string(), local_with_script).unwrap(); + } + + #[test] + fn nitro_enclave_does_not_require_tdx_mr_config() -> Result<()> { + let compose_hash = [0u8; 32]; + let gpu_policy_hash = [0u8; 32]; + let app_id = [0u8; 20]; + let instance_id = [0u8; 20]; + let local = LocalMrConfigValues { + compose_hash: &compose_hash, + gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &[], + app_id: &app_id, + instance_id: &instance_id, + key_provider: KeyProviderKind::None, + key_provider_id: &[], + }; + + verify_mr_config_id_for_mode(TeeVariant::DstackNitroEnclave, local) + } } diff --git a/dstack/dstack-util/tests/fixtures/gpu_attestation_h100.json b/dstack/dstack-util/tests/fixtures/gpu_attestation_h100.json new file mode 100644 index 000000000..c8e52e018 --- /dev/null +++ b/dstack/dstack-util/tests/fixtures/gpu_attestation_h100.json @@ -0,0 +1,67 @@ +{ + "claims": [ + { + "dbgstat": "disabled", + "eat_nonce": "1111111111111111111111111111111111111111111111111111111111111111", + "hwmodel": "GH100 A01 GSP BROM", + "measres": "success", + "oemid": "5703", + "secboot": true, + "ueid": "457723480349051719100670602112458195243836813799", + "x-nvidia-device-type": "gpu", + "x-nvidia-gpu-arch-check": true, + "x-nvidia-gpu-attestation-report-cert-chain": { + "x-nvidia-cert-expiration-date": "9999-12-31T23:59:59Z", + "x-nvidia-cert-ocsp-nonce-matches": true, + "x-nvidia-cert-ocsp-response-valid": true, + "x-nvidia-cert-ocsp-status": "good", + "x-nvidia-cert-revocation-reason": null, + "x-nvidia-cert-status": "valid" + }, + "x-nvidia-gpu-attestation-report-cert-chain-fwid-match": true, + "x-nvidia-gpu-attestation-report-nonce-match": true, + "x-nvidia-gpu-attestation-report-parsed": true, + "x-nvidia-gpu-attestation-report-signature-verified": true, + "x-nvidia-gpu-claims-version": "3.0", + "x-nvidia-gpu-driver-rim-cert-chain": { + "x-nvidia-cert-expiration-date": "2028-04-23T05:31:21Z", + "x-nvidia-cert-ocsp-nonce-matches": true, + "x-nvidia-cert-ocsp-response-valid": true, + "x-nvidia-cert-ocsp-status": "good", + "x-nvidia-cert-revocation-reason": null, + "x-nvidia-cert-status": "valid" + }, + "x-nvidia-gpu-driver-rim-fetched": true, + "x-nvidia-gpu-driver-rim-measurements-available": true, + "x-nvidia-gpu-driver-rim-signature-verified": true, + "x-nvidia-gpu-driver-rim-version-match": true, + "x-nvidia-gpu-driver-version": "580.159.03", + "x-nvidia-gpu-vbios-index-no-conflict": true, + "x-nvidia-gpu-vbios-rim-cert-chain": { + "x-nvidia-cert-expiration-date": "2027-04-30T23:59:59Z", + "x-nvidia-cert-ocsp-nonce-matches": true, + "x-nvidia-cert-ocsp-response-valid": true, + "x-nvidia-cert-ocsp-status": "good", + "x-nvidia-cert-revocation-reason": null, + "x-nvidia-cert-status": "valid" + }, + "x-nvidia-gpu-vbios-rim-fetched": true, + "x-nvidia-gpu-vbios-rim-measurements-available": true, + "x-nvidia-gpu-vbios-rim-signature-verified": true, + "x-nvidia-gpu-vbios-rim-version-match": true, + "x-nvidia-gpu-vbios-version": "96.00.D9.00.01", + "x-nvidia-mismatch-measurement-records": null + } + ], + "detached_eat": [ + [ + "JWT", + "eyJhbGciOiJub25lIn0.eyJlYXRfbm9uY2UiOiIxMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExIiwiZXhwIjoxNzg0MjExNDYyLCJpYXQiOjE3ODQyMDc4NjIsImlzcyI6Ik5WQVQtTE9DQUwtVkVSSUZJRVIiLCJqdGkiOiI2MjcwZTgxMTVhNWMxOTExODNjNjg0ZWY4YzFkYWRmYzYxYzYzMjQyZGUwOGY0MjQzOGU3M2NmMjNjODBkMTg1Iiwic3ViIjoiTlZJRElBLVBMQVRGT1JNLUFUVEVTVEFUSU9OIiwic3VibW9kcyI6eyJHUFUtMCI6WyJESUdFU1QiLFsiU0hBMjU2IiwiMDNkZTczYjliMTU5OTM2OTNkZmMyMzY3Y2QyZjNmMDE3ZWVkYjQ2ZWZhNzY3Yzc5ZGJkODY3ODZlMGQ3OWZlYSJdXX0sIngtbnZpZGlhLW92ZXJhbGwtYXR0LXJlc3VsdCI6dHJ1ZSwieC1udmlkaWEtdmVyIjoiMy4wIn0." + ], + { + "GPU-0": "eyJhbGciOiJub25lIn0.eyJkYmdzdGF0IjoiZGlzYWJsZWQiLCJlYXRfbm9uY2UiOiIxMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExIiwiZXhwIjoxNzg0MjExNDYyLCJod21vZGVsIjoiR0gxMDAgQTAxIEdTUCBCUk9NIiwiaWF0IjoxNzg0MjA3ODYyLCJpc3MiOiJOVkFULUxPQ0FMLVZFUklGSUVSIiwianRpIjoiYTU5ZWQ5ODAwYThhNjFhZDc5NjVlNmI1MGRmMjEyODVlNjMxZTg5ZWNlZTM5NmMwZjc1ZDkwNjIwNWI2MDUzYiIsIm1lYXNyZXMiOiJzdWNjZXNzIiwib2VtaWQiOiI1NzAzIiwic2VjYm9vdCI6dHJ1ZSwidWVpZCI6IjQ1NzcyMzQ4MDM0OTA1MTcxOTEwMDY3MDYwMjExMjQ1ODE5NTI0MzgzNjgxMzc5OSIsIngtbnZpZGlhLWRldmljZS10eXBlIjoiZ3B1IiwieC1udmlkaWEtZ3B1LWFyY2gtY2hlY2siOnRydWUsIngtbnZpZGlhLWdwdS1hdHRlc3RhdGlvbi1yZXBvcnQtY2VydC1jaGFpbiI6eyJ4LW52aWRpYS1jZXJ0LWV4cGlyYXRpb24tZGF0ZSI6Ijk5OTktMTItMzFUMjM6NTk6NTlaIiwieC1udmlkaWEtY2VydC1vY3NwLW5vbmNlLW1hdGNoZXMiOnRydWUsIngtbnZpZGlhLWNlcnQtb2NzcC1yZXNwb25zZS12YWxpZCI6dHJ1ZSwieC1udmlkaWEtY2VydC1vY3NwLXN0YXR1cyI6Imdvb2QiLCJ4LW52aWRpYS1jZXJ0LXJldm9jYXRpb24tcmVhc29uIjpudWxsLCJ4LW52aWRpYS1jZXJ0LXN0YXR1cyI6InZhbGlkIn0sIngtbnZpZGlhLWdwdS1hdHRlc3RhdGlvbi1yZXBvcnQtY2VydC1jaGFpbi1md2lkLW1hdGNoIjp0cnVlLCJ4LW52aWRpYS1ncHUtYXR0ZXN0YXRpb24tcmVwb3J0LW5vbmNlLW1hdGNoIjp0cnVlLCJ4LW52aWRpYS1ncHUtYXR0ZXN0YXRpb24tcmVwb3J0LXBhcnNlZCI6dHJ1ZSwieC1udmlkaWEtZ3B1LWF0dGVzdGF0aW9uLXJlcG9ydC1zaWduYXR1cmUtdmVyaWZpZWQiOnRydWUsIngtbnZpZGlhLWdwdS1jbGFpbXMtdmVyc2lvbiI6IjMuMCIsIngtbnZpZGlhLWdwdS1kcml2ZXItcmltLWNlcnQtY2hhaW4iOnsieC1udmlkaWEtY2VydC1leHBpcmF0aW9uLWRhdGUiOiIyMDI4LTA0LTIzVDA1OjMxOjIxWiIsIngtbnZpZGlhLWNlcnQtb2NzcC1ub25jZS1tYXRjaGVzIjp0cnVlLCJ4LW52aWRpYS1jZXJ0LW9jc3AtcmVzcG9uc2UtdmFsaWQiOnRydWUsIngtbnZpZGlhLWNlcnQtb2NzcC1zdGF0dXMiOiJnb29kIiwieC1udmlkaWEtY2VydC1yZXZvY2F0aW9uLXJlYXNvbiI6bnVsbCwieC1udmlkaWEtY2VydC1zdGF0dXMiOiJ2YWxpZCJ9LCJ4LW52aWRpYS1ncHUtZHJpdmVyLXJpbS1mZXRjaGVkIjp0cnVlLCJ4LW52aWRpYS1ncHUtZHJpdmVyLXJpbS1tZWFzdXJlbWVudHMtYXZhaWxhYmxlIjp0cnVlLCJ4LW52aWRpYS1ncHUtZHJpdmVyLXJpbS1zaWduYXR1cmUtdmVyaWZpZWQiOnRydWUsIngtbnZpZGlhLWdwdS1kcml2ZXItcmltLXZlcnNpb24tbWF0Y2giOnRydWUsIngtbnZpZGlhLWdwdS1kcml2ZXItdmVyc2lvbiI6IjU4MC4xNTkuMDMiLCJ4LW52aWRpYS1ncHUtdmJpb3MtaW5kZXgtbm8tY29uZmxpY3QiOnRydWUsIngtbnZpZGlhLWdwdS12Ymlvcy1yaW0tY2VydC1jaGFpbiI6eyJ4LW52aWRpYS1jZXJ0LWV4cGlyYXRpb24tZGF0ZSI6IjIwMjctMDQtMzBUMjM6NTk6NTlaIiwieC1udmlkaWEtY2VydC1vY3NwLW5vbmNlLW1hdGNoZXMiOnRydWUsIngtbnZpZGlhLWNlcnQtb2NzcC1yZXNwb25zZS12YWxpZCI6dHJ1ZSwieC1udmlkaWEtY2VydC1vY3NwLXN0YXR1cyI6Imdvb2QiLCJ4LW52aWRpYS1jZXJ0LXJldm9jYXRpb24tcmVhc29uIjpudWxsLCJ4LW52aWRpYS1jZXJ0LXN0YXR1cyI6InZhbGlkIn0sIngtbnZpZGlhLWdwdS12Ymlvcy1yaW0tZmV0Y2hlZCI6dHJ1ZSwieC1udmlkaWEtZ3B1LXZiaW9zLXJpbS1tZWFzdXJlbWVudHMtYXZhaWxhYmxlIjp0cnVlLCJ4LW52aWRpYS1ncHUtdmJpb3MtcmltLXNpZ25hdHVyZS12ZXJpZmllZCI6dHJ1ZSwieC1udmlkaWEtZ3B1LXZiaW9zLXJpbS12ZXJzaW9uLW1hdGNoIjp0cnVlLCJ4LW52aWRpYS1ncHUtdmJpb3MtdmVyc2lvbiI6Ijk2LjAwLkQ5LjAwLjAxIiwieC1udmlkaWEtbWlzbWF0Y2gtbWVhc3VyZW1lbnQtcmVjb3JkcyI6bnVsbH0." + } + ], + "result_code": 0, + "result_message": "Ok" +} diff --git a/dstack/gateway/Cargo.toml b/dstack/gateway/Cargo.toml index ebe4d7b35..e682bb690 100644 --- a/dstack/gateway/Cargo.toml +++ b/dstack/gateway/Cargo.toml @@ -10,6 +10,7 @@ edition.workspace = true license.workspace = true [dependencies] +dstack-attest.workspace = true arc-swap.workspace = true rocket = { workspace = true, features = ["mtls", "json"] } tracing.workspace = true @@ -30,8 +31,8 @@ hickory-resolver.workspace = true pin-project.workspace = true serde_json.workspace = true rand.workspace = true -git-version.workspace = true -ra-rpc = { workspace = true, features = ["client", "rocket"] } +dstack-build-info.workspace = true +ra-rpc = { workspace = true, features = ["rocket"] } dstack-gateway-rpc.workspace = true certbot.workspace = true bytes.workspace = true @@ -40,7 +41,6 @@ smallvec.workspace = true futures.workspace = true cmd_lib.workspace = true load_config.workspace = true -dstack-kms-rpc.workspace = true ra-tls.workspace = true dstack-guest-agent-rpc.workspace = true http-client = { workspace = true, features = ["prpc"] } @@ -60,18 +60,26 @@ tdx-attest.workspace = true flate2.workspace = true uuid = { workspace = true, features = ["v4"] } rmp-serde.workspace = true +rmp.workspace = true or-panic.workspace = true base64.workspace = true -subtle.workspace = true +dstack-api-auth.workspace = true cached-cell.workspace = true [target.'cfg(unix)'.dependencies] -nix = { workspace = true, features = ["resource"] } - -[[bin]] -name = "gen_debug_key" -path = "src/gen_debug_key.rs" +nix = { workspace = true, features = ["resource", "fs", "socket", "zerocopy"] } +ktls.workspace = true +libc.workspace = true +socket2.workspace = true [dev-dependencies] insta.workspace = true +# The test-only CA/leaf helpers. A dev-dependency on the same crate the normal +# dependency points at, so the feature is enabled for tests and not for the binary. +ra-tls = { workspace = true, features = ["test-pki"] } +rmpv.workspace = true tempfile.workspace = true +wavekv-v1 = { package = "wavekv", version = "=1.0.0" } +# `test-util` gives the idle-watchdog tests a paused clock, so they assert on +# the window without waiting for it in wall-clock time. +tokio = { workspace = true, features = ["test-util"] } diff --git a/dstack/gateway/README.md b/dstack/gateway/README.md new file mode 100644 index 000000000..af79ee75f --- /dev/null +++ b/dstack/gateway/README.md @@ -0,0 +1,43 @@ +# dstack-gateway + +The reverse proxy that gives deployed dstack apps zero-trust network access. It +terminates TLS, provisions certificates over ACME DNS-01, and routes traffic to +CVMs across a WireGuard mesh it manages itself. + +## Ingress mapping + +Apps are addressed as `<id>[-[<port>][s|g]].<base_domain>`: + +| Form | Behaviour | +| --- | --- | +| `<id>.<base_domain>` | TLS terminated at the gateway, forwarded as TCP | +| `<id>-<port>s.<base_domain>` | TLS passthrough to the app | +| `<id>-<port>g.<base_domain>` | TLS terminated, forwarded as HTTP/2 (gRPC) | + +`<id>` is the app ID or the instance ID; `<port>` defaults to 80. + +## Configuration + +`gateway.toml` holds the base domain, the certificate and WireGuard settings, +and the admin API credentials. Operator-facing admin RPCs are served on a +separate listener behind the shared HTTP authenticator, the same way the KMS and +VMM expose theirs. + +## Running + +Build and run from the workspace root: + +```bash +cargo build --release -p dstack-gateway +sudo ./target/release/dstack-gateway -c gateway.toml +``` + +The gateway needs `CAP_NET_ADMIN` to manage its WireGuard interface, hence +`sudo`. + +## Further reading + +- [Cluster deployment](docs/cluster-deployment.md) — running a replicated gateway +- [Builder image](dstack-app/builder/README.md) — reproducible container image +- [Deployment guide](../../docs/deployment.md) +- [Security guide](../../docs/security-guide/security-guide.md) diff --git a/dstack/gateway/docs/cluster-deployment.md b/dstack/gateway/docs/cluster-deployment.md index 191d4c0ef..3d248a048 100644 --- a/dstack/gateway/docs/cluster-deployment.md +++ b/dstack/gateway/docs/cluster-deployment.md @@ -164,9 +164,57 @@ This allows a simple deployment order: 3. Gateway-2 fetches peers from gateway-1 and starts syncing 4. Gateway-1 auto-discovers gateway-2 from the incoming sync request +### 2.5 Consistency Model and Operational Constraints + +WaveKV provides per-key, last-writer-wins eventual consistency. It does not +provide transactions, compare-and-swap, quorum writes, or linearizable reads. +Operate the Gateway cluster with the following constraints: + +- A `node_id` identifies one sequence-number writer. It must be unique among + all live nodes and must never be used concurrently by a replacement node. + UUID conflict detection catches many accidental reuses, but it is not a node + ID lease. Permanently stop the old writer before reusing its ID. +- Keep each node's WaveKV data directory on persistent storage. If an existing + node loses that directory, do not let it accept writes under its old + `node_id` until it has recovered from at least one up-to-date peer. If no peer + is reachable, restore the directory from backup or provision the node with a + new `node_id`; starting an isolated writer from an empty sequence history can + reuse sequence numbers already observed by the cluster. +- Keep system clocks synchronized. Conflict resolution uses wall-clock time, + with node ID and sequence number as tie-breakers. Instance and telemetry + records more than five minutes in the future are ignored, but a clock that is + behind can still cause a legitimate update to lose to an older value. +- Client IP allocation is local to each Gateway. The configured client address + pools must not overlap. +- WireGuard public-key uniqueness is not an atomic cluster-wide reservation. + During a partition, two nodes can register the same key for different + instances. After synchronization, every Gateway deterministically routes + only the conflict winner, but the losing CVM can be temporarily routable + before convergence. Workloads must retry registration and tolerate this + reconciliation. +- Certificate renewal and ACME credential-rotation coordination uses + best-effort WaveKV records, not a distributed mutex. A partition can allow + more than one node to perform the operation. These operations must remain + idempotent, and external DNS/ACME side effects must tolerate duplicate work. +- A successful WaveKV sync or matching digest describes replicated KV state, + not instantaneous data-plane state. The Gateway asynchronously reconciles + its in-memory `ProxyState`, WireGuard peers, certificates, and other + materialized views. Monitoring and maintenance automation should allow a + reconciliation interval and verify the relevant data-plane/admin endpoint, + rather than treating the sync result alone as readiness. +- Concurrent administrator updates to the same key have LWW semantics rather + than causal ordering. Serialize security-sensitive configuration changes at + the operational layer when losing an update would be unsafe. + +For a brand-new node, an empty local store and temporarily empty peer list are +expected. The stricter recovery rule above applies when a previously active +node loses its store while retaining its identity. + > Note: `bootnode` is only used for initial discovery. Once peers are discovered, they are persisted in the KV store and survive restarts. -### 2.5 Configuration File Examples +### 2.6 Configuration File Examples + +> **Note:** A non-empty `rpc_domain` makes the gateway request its RPC TLS key and certificate from the local dstack Guest Agent. Ensure `/var/run/dstack/dstack.sock` is available, or set `DSTACK_AGENT_ADDRESS` to another Guest Agent endpoint. Set `rpc_domain = ""` when supplying pre-generated certificates. gateway-1.toml: @@ -184,7 +232,6 @@ ca_certs = "/var/lib/gateway/certs/gateway-ca.cert" mandatory = false [core] -kms_url = "https://kms.demo.dstack.org" rpc_domain = "rpc.gateway-1.demo.dstack.org" [core.admin] @@ -194,7 +241,6 @@ address = "0.0.0.0" [core.debug] insecure_enable_debug_rpc = true -insecure_skip_attestation = false port = 9015 address = "0.0.0.0" @@ -240,7 +286,6 @@ ca_certs = "/var/lib/gateway/certs/gateway-ca.cert" mandatory = false [core] -kms_url = "https://kms.demo.dstack.org" rpc_domain = "rpc.gateway-2.demo.dstack.org" [core.sync] @@ -269,7 +314,7 @@ listen_port = 9014 external_port = 443 ``` -### 2.6 Single-Host Deployment Notes +### 2.7 Single-Host Deployment Notes If you run multiple gateway nodes on the same physical host (for example, multiple CVMs on one teepod / dstack-vmm host), the default example ports above will conflict. You must assign distinct host-facing ports per node. @@ -287,9 +332,9 @@ Important: - Each gateway VM must have a **unique name** when deployed to the same VMM (e.g., `dstack-gateway-1` and `dstack-gateway-2`) - Create DNS records for the RPC hostnames before bootstrapping the cluster -### 2.7 Verify Cluster Sync +### 2.8 Verify Cluster Sync -The admin API requires a bearer token (see `core.admin.admin_token` in `gateway.toml`, +The admin API requires a bearer token (see `core.admin.auth_token` in `gateway.toml`, or the `ADMIN_API_TOKEN` env injected by `deploy-to-vmm.sh`). Export it once: ```bash diff --git a/dstack/gateway/dstack-app/builder/Dockerfile b/dstack/gateway/dstack-app/builder/Dockerfile index e91b00486..a4446a7d9 100644 --- a/dstack/gateway/dstack-app/builder/Dockerfile +++ b/dstack/gateway/dstack-app/builder/Dockerfile @@ -9,7 +9,7 @@ ARG DSTACK_REV ARG DSTACK_SRC_URL=https://github.com/Dstack-TEE/dstack.git WORKDIR /build RUN ./pin-packages.sh ./builder-pinned-packages.txt -RUN apt-get update && \ +RUN apt-get update -o APT::Update::Error-Mode=any && \ apt-get install -y --no-install-recommends \ git \ build-essential \ @@ -31,7 +31,7 @@ COPY --from=build-shared pin-packages.sh /build/ COPY ./shared/pinned-packages.txt /build/ WORKDIR /build RUN ./pin-packages.sh ./pinned-packages.txt && \ - apt-get update && \ + apt-get update -o APT::Update::Error-Mode=any && \ apt-get install -y --no-install-recommends \ git \ wireguard-tools \ @@ -41,6 +41,11 @@ RUN ./pin-packages.sh ./pinned-packages.txt && \ && rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache COPY --from=gateway-builder /build/repo/dstack/target/x86_64-unknown-linux-musl/release/dstack-gateway /usr/local/bin/dstack-gateway COPY --from=gateway-builder /build/.GIT_REV /etc/ +# The OCI metadata is generated once by build-lib.sh and injected here, as +# the config labels and as the manifest annotations, so the three views of +# the image can never disagree. +ARG IMAGE_METADATA +RUN mkdir -p /etc/dstack-gateway && printf '%s\n' "${IMAGE_METADATA}" > /etc/dstack-gateway/build-info WORKDIR /app COPY entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh diff --git a/dstack/gateway/dstack-app/builder/README.md b/dstack/gateway/dstack-app/builder/README.md index b5387fa93..18e074452 100644 --- a/dstack/gateway/dstack-app/builder/README.md +++ b/dstack/gateway/dstack-app/builder/README.md @@ -1,55 +1,88 @@ -# dstack KMS Builder +# dstack Gateway Builder -This directory contains the necessary files to build and run the dstack-kms Docker image for development. +This directory contains the files needed to build the dstack-gateway Docker +image reproducibly. ## Overview -The builder creates a Docker image that includes: -- The dstack-kms service compiled from Rust source code -- Command line tool dstack-acpi-tables for generating ACPI tables for dstack CVM +The builder produces a Docker image containing the dstack-gateway service +compiled from Rust source. The build pins its base images by digest, pins the +Debian package set (`shared/pinned-packages.txt`), and normalizes timestamps, so +the same revision always yields the same image. ## Prerequisites -- Docker with BuildKit support (v20.10.0+) +- Docker with Buildx v0.13.0+ (the script creates a BuildKit v0.20.2 builder) - Git ## Building the Image -To build the KMS Docker image, use the provided `build-image.sh` script: - ```bash -./build-image.sh <image-name>[:<tag>] +./build-image.sh <image-name>[:<tag>]... ``` For example: + ```bash -./build-image.sh kvin/kms +./build-image.sh my-org/dstack-gateway:dev ``` -## Running the Built Image +Optional environment variables: + +| Variable | Purpose | +| --- | --- | +| `GIT_REV` | Revision to build (default `HEAD`) | +| `IMAGE_VERSION` | Version recorded in the image metadata (default `dev`) | +| `IMAGE_SOURCE_URL` | Repository URL recorded in the image metadata | +| `NO_CACHE` | Set to any value to build without the layer cache | +| `OCI_TAR` | Also write an OCI archive here, for digest comparison | +| `METADATA_FILE` | Write the validated OCI manifest digest and build metadata here | +| `PUSH` | Set to any value to push the tags instead of only loading them | + +Publication and OCI export happen only after both package lists pass validation. +`NO_CACHE` applies to the validation builds; export then reuses their cached result. +Manual release workflows require an existing component release tag and check out +that tag, rather than building the branch selected in the workflow UI. + +## Reproducing a released image + +Release CI runs this same script, so a published image can be rebuilt and +checked digest-for-digest. Use a clean checkout of the release commit so the +Dockerfile, package lists, shared scripts and copied files also match the release. +From the repository root (fetch the tag first if it is not available locally): + +```bash +git switch --detach "gateway-v0.6.0^{commit}" +cd dstack/gateway/dstack-app/builder + +GIT_REV=HEAD \ +IMAGE_VERSION=0.6.0 \ +IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack \ +OCI_TAR=/tmp/gateway.oci.tar \ + ./build-image.sh dstacktee/dstack-gateway:0.6.0 -### Using Docker Compose - -The easiest way to run the KMS service is using the provided `docker-compose.yaml`: - -```yaml -services: - kms: - image: kvin/kms - ports: - - "8003:8000" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ./kms:/kms - environment: - - IMAGE_DOWNLOAD_URL=${IMAGE_DOWNLOAD_URL:-http://localhost:8001/mr_{OS_IMAGE_HASH}.tar.gz} - - AUTH_TYPE=dev - - DEV_DOMAIN=kms.1022.dstack.org - - QUOTE_ENABLED=false +python3 -c 'import json,tarfile;t=tarfile.open("/tmp/gateway.oci.tar");print(json.load(t.extractfile("index.json"))["manifests"][0]["digest"])' ``` -To start the service: +`IMAGE_VERSION` is part of the image metadata, so it must match the release for +the digests to match. The printed digest is what the registry reports for +`dstacktee/dstack-gateway:0.6.0`; compare it with: ```bash -docker-compose up +docker buildx imagetools inspect dstacktee/dstack-gateway:0.6.0 --format '{{.Manifest.Digest}}' ``` + +## Image metadata + +The image carries its provenance as OCI metadata in three places, all generated +from one definition in `dstack/build/shared/build-lib.sh`: + +- config labels — `docker inspect -f '{{json .Config.Labels}}' <image>` +- manifest annotations — `docker buildx imagetools inspect <image>` +- `/etc/dstack-gateway/build-info` inside the image, readable from within the CVM + +## Running the Built Image + +The gateway is normally deployed as a dstack app; see +[`../deploy-to-vmm.sh`](../deploy-to-vmm.sh) and the +[gateway README](../../README.md). diff --git a/dstack/gateway/dstack-app/builder/build-image.sh b/dstack/gateway/dstack-app/builder/build-image.sh index 3d86546f1..1b1992c57 100755 --- a/dstack/gateway/dstack-app/builder/build-image.sh +++ b/dstack/gateway/dstack-app/builder/build-image.sh @@ -11,18 +11,20 @@ REPO_ROOT=$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel) CONTEXT_DIR="$SCRIPT_DIR" SHARED_DIR="$SCRIPT_DIR/shared" DOCKERFILE="$SCRIPT_DIR/Dockerfile" +export CONTEXT_DIR DOCKERFILE +# shellcheck source=/dev/null source "$REPO_ROOT/dstack/build/shared/build-lib.sh" -NAME=${1:-} -if [ -z "$NAME" ]; then - echo "Usage: $0 <image-name>[:<tag>]" >&2 +if [ "$#" -eq 0 ]; then + echo "Usage: $0 <image-name>[:<tag>]..." >&2 exit 1 fi +TAGS=$(printf '%s\n' "$@") NO_CACHE=${NO_CACHE:-} GIT_REV=${GIT_REV:-HEAD} -GIT_REV=$(git -C "$REPO_ROOT" rev-parse "$GIT_REV") +GIT_REV=$(git -C "$REPO_ROOT" rev-parse --verify "${GIT_REV}^{commit}") DSTACK_SRC_URL=${DSTACK_SRC_URL:-https://github.com/Dstack-TEE/dstack.git} ensure_buildkit @@ -30,7 +32,10 @@ ensure_buildkit touch "$SHARED_DIR/builder-pinned-packages.txt" touch "$SHARED_DIR/pinned-packages.txt" -docker_build "$NAME" "" "$SHARED_DIR/pinned-packages.txt" -docker_build "gateway-builder-temp" "gateway-builder" "$SHARED_DIR/builder-pinned-packages.txt" +METADATA=$(image_metadata \ + "dstack-gateway" \ + "Gateway service for dstack confidential applications" \ + "dstack/gateway" \ + "dstack/gateway/README.md") -check_clean_tree "$SHARED_DIR" +build_component "$TAGS" "gateway-builder-temp" "gateway-builder" "$SHARED_DIR" "$METADATA" diff --git a/dstack/gateway/dstack-app/builder/entrypoint.sh b/dstack/gateway/dstack-app/builder/entrypoint.sh index 1a3811ad0..179d26ad1 100755 --- a/dstack/gateway/dstack-app/builder/entrypoint.sh +++ b/dstack/gateway/dstack-app/builder/entrypoint.sh @@ -51,7 +51,7 @@ fi # Sync is always enabled when NODE_ID > 0. Peer auto-discovery works via incoming # sync connections: when another node syncs to us, we learn about it automatically -# through WaveKV's handle_sync, which auto-adds the sender as a peer. +# through WaveKV's envelope handler, which auto-adds the sender as a peer. # BOOTNODE_URL is optional — it speeds up initial discovery but is not required. SYNC_ENABLED=$([ "$NODE_ID" -gt 0 ] && echo "true" || echo "false") @@ -95,7 +95,7 @@ sync_connections_interval = "${SYNC_CONNECTIONS_INTERVAL:-30s}" enabled = true address = "${ADMIN_LISTEN_ADDR:-0.0.0.0}" port = ${ADMIN_LISTEN_PORT:-8001} -admin_token = "${ADMIN_API_TOKEN}" +auth_token = "${ADMIN_API_TOKEN}" [core.wg] public_key = "$PUBLIC_KEY" @@ -114,7 +114,6 @@ tls_versions = ["1.2"] listen_addr = "0.0.0.0" listen_port = "${PROXY_LISTEN_PORT:-443}" connect_top_n = 3 -localhost_enabled = false app_address_ns_compat = true workers = ${PROXY_WORKERS:-32} max_connections_per_app = ${MAX_CONNECTIONS_PER_APP:-0} @@ -124,6 +123,7 @@ inbound_pp_enabled = ${INBOUND_PP_ENABLED:-false} connect = "${TIMEOUT_CONNECT:-5s}" handshake = "${TIMEOUT_HANDSHAKE:-5s}" cache_top_n = "${TIMEOUT_CACHE_TOP_N:-30s}" +handshake_stale = "${TIMEOUT_HANDSHAKE_STALE:-30m}" dns_resolve = "${TIMEOUT_DNS_RESOLVE:-5s}" data_timeout_enabled = ${TIMEOUT_DATA_ENABLED:-true} idle = "${TIMEOUT_IDLE:-10m}" @@ -132,6 +132,13 @@ shutdown = "${TIMEOUT_SHUTDOWN:-5s}" total = "${TIMEOUT_TOTAL:-5h}" pp_header = "${TIMEOUT_PP_HEADER:-5s}" +[core.proxy.health_check] +enabled = ${HEALTH_CHECK_ENABLED:-true} +interval = "${HEALTH_CHECK_INTERVAL:-5s}" +timeout = "${HEALTH_CHECK_TIMEOUT:-2s}" +concurrency = ${HEALTH_CHECK_CONCURRENCY:-16} +failure_threshold = ${HEALTH_CHECK_FAILURE_THRESHOLD:-2} + [core.proxy.port_policy_fetch] timeout = "${PORT_POLICY_FETCH_TIMEOUT:-10s}" max_retries = ${PORT_POLICY_FETCH_MAX_RETRIES:-5} diff --git a/dstack/gateway/gateway.toml b/dstack/gateway/gateway.toml index a7383abb8..07be1c145 100644 --- a/dstack/gateway/gateway.toml +++ b/dstack/gateway/gateway.toml @@ -11,11 +11,28 @@ log_level = "info" address = "127.0.0.1:8010" [core] -kms_url = "" # auto set soft ulimit to hard ulimit set_ulimit = true rpc_domain = "" +# Required before any root_ca entry is accepted. Development/CI only; +# production should retain vendor roots. +[core.attestation] +insecure_allow_external_trust_anchors = false + +[core.attestation.urls] +# pccs = "https://pccs.phala.network" +# amd_kds = "https://kdsintf.amd.com/vcek/v1" + +[core.attestation.root_ca] +# tdx = "/etc/dstack/roots/intel-sgx-root-ca.der" +# gcp_tpm = "/etc/dstack/roots/gcp-tpm-root-ca.pem" +# aws_nitro_enclave = "/etc/dstack/roots/aws-nitro-enclave-root-ca.pem" +# aws_nitro_tpm = "/etc/dstack/roots/aws-nitro-tpm-root-ca.pem" +# sev_snp_milan = "/etc/dstack/roots/amd-milan-ark.pem" +# sev_snp_genoa = "/etc/dstack/roots/amd-genoa-ark.pem" +# sev_snp_turin = "/etc/dstack/roots/amd-turin-ark.pem" + [core.auth] enabled = false url = "http://localhost/app-auth" @@ -28,16 +45,21 @@ address = "127.0.0.1:8011" # be supplied via the `DSTACK_GATEWAY_ADMIN_TOKEN` or `ADMIN_API_TOKEN` env # vars. Clients send it as `Authorization: Bearer <token>`, `X-Admin-Token`, # or (GET only, for dashboard links) `?token=...`. Required unless -# `insecure_no_auth = true`. -admin_token = "" +# `insecure_no_auth = true`. (The legacy key `admin_token` is still accepted.) +auth_token = "" +# Optional Apache htpasswd file for HTTP Basic authentication. +htpasswd_file = "" # Development/testing escape hatch only. Never enable this on an admin # interface that is reachable from the network. insecure_no_auth = false [core.debug] insecure_enable_debug_rpc = false -insecure_skip_attestation = false -key_file = "debug_key.json" +# Route the app address "localhost" to 127.0.0.1 on the gateway host. Off by +# default: the app address also comes from the _dstack-app-address TXT record of +# arbitrary custom domains, so enabling this lets any DNS zone owner reach the +# gateway's own loopback on a port of their choosing, bypassing port_policy. +insecure_localhost_backend = false address = "127.0.0.1:8012" [core.wg] @@ -57,18 +79,60 @@ tls_versions = ["1.2"] listen_addr = "0.0.0.0" listen_port = 8443 agent_port = 8090 -buffer_size = 8192 +buffer_size = 65536 # number of hosts to try to connect to connect_top_n = 3 -localhost_enabled = false app_address_ns_prefix = "_dstack-app-address" app_address_ns_compat = true workers = 32 +# One runtime + SO_REUSEPORT listener per worker (thread-per-core). Linux only. +thread_per_core = true +# Hand new connections to a less loaded core when SO_REUSEPORT skews them. +connection_rebalance = true external_port = 443 # Maximum concurrent connections per app. 0 means unlimited. max_connections_per_app = 2000 # Whether to read PROXY protocol from inbound connections (e.g. from Cloudflare). inbound_pp_enabled = false +# splice(2) zero-copy relaying for the TLS-passthrough path (Linux only), and +# kernel TLS offload for the terminate path, are both configured by an optional +# section. Omitting the section disables the optimisation; an empty section +# engages it from the first byte; `after_bytes` / `after_duration` are two +# independent gates and whichever fires first engages it. +# +# [core.proxy.tcp_splice] +# # Bulk transfers trip this within milliseconds. +# after_bytes = 65536 +# # Long-lived low-rate streams (e.g. LLM token streaming) never accumulate +# # bytes fast enough for the gate above, so they need this one instead. +# after_duration = "5s" +# # Park a relay's pipe in the pool while it waits for the next chunk. A spliced +# # connection otherwise pins 4 descriptors for its whole lifetime, so bursty +# # traffic (a token every 25 ms) holds them >99% idle. +# release_idle_pipes = true +# +# kTLS hands session keys to the kernel; see config.rs for the security note. +# Gated offload also requires [core.proxy.tcp_splice]. +# +# [core.proxy.ktls] +# after_bytes = 65536 + +# Application-level health polling. The gateway asks each CVM's guest agent +# whether the app is serving, and keeps instances that say no out of app-id +# load balancing. Instances stay reachable by instance id regardless. +# +# Safe to leave on: only CVMs whose app-compose sets `requirements.health_check` +# are polled at all, and if every instance of an app reports unhealthy the +# gateway routes to all of them anyway rather than blackholing the app. +# Note the two are different keys with similar names: `enabled` below switches +# polling on for this gateway, `requirements.health_check` is how an app asks +# to be polled. +[core.proxy.health_check] +enabled = true +interval = "5s" +timeout = "2s" +concurrency = 16 +failure_threshold = 2 [core.proxy.port_policy_fetch] # Background lazy-fetch of port_policy from legacy CVM agents. @@ -88,6 +152,8 @@ handshake = "5s" # Timeout for top n hosts selection cache_top_n = "30s" +# Maximum WireGuard handshake age for a healthy upstream instance. +handshake_stale = "30m" # Timeout for DNS TXT record resolution (app address lookup). dns_resolve = "5s" @@ -125,7 +191,35 @@ bootnode = "" data_dir = "/dstack-gateway/data" # Interval for periodic persistence of WaveKV data (e.g., "5s", "1m", "1h") persist_interval = "5m" +# How long a KV write may sit in the page cache before the write-ahead log is +# forced to disk. +# +# An fsync costs about 100us on an NVMe host and milliseconds on the virtio disk +# a CVM gets, and it runs under the store lock, so paying one per write bounds +# how fast this gateway can accept registrations and stalls its readers while it +# does. Deferring it bounds what losing the *machine* costs — writes made in the +# last window — and changes nothing about losing the process: a panic, an OOM +# kill or a restart loses nothing either way, because the bytes are already with +# the kernel. +# +# A cluster recovers the window from its peers on restart. A single-node gateway +# does not, which is the case for setting this to "0s". +# +# Note that "0s" is the strictest setting here, not the cheapest: no window +# means every write is forced before it returns, as releases before WaveKV 2.1 +# always did. That is the opposite of the "*_interval" settings either side of +# it, where zero turns the work off — which is why this one is a window. +wal_sync_window = "5s" # Enable periodic sync of instance connections to KV store sync_connections_enabled = true # Interval for syncing instance connections to KV store sync_connections_interval = "30s" +# Collect tombstones every peer has already acknowledged once every this many +# replicated writes, per store. Zero keeps every delete on disk for the life of +# the deployment. The trigger counts replicated writes rather than reading a +# clock so that every node in a cluster collects in the same window -- the +# state digest covers tombstones, so nodes collecting on their own phase +# report a mismatch and pay a full re-exchange. An operator override stored in +# the KV itself (SetTombstoneGcConfig) replicates everywhere and takes +# precedence over this per-node default. +tombstone_gc_writes = 10000 diff --git a/dstack/gateway/rpc/proto/gateway_rpc.proto b/dstack/gateway/rpc/proto/gateway_rpc.proto index 046b731ed..5a59732de 100644 --- a/dstack/gateway/rpc/proto/gateway_rpc.proto +++ b/dstack/gateway/rpc/proto/gateway_rpc.proto @@ -12,6 +12,20 @@ package gateway; message RegisterCvmRequest { // The public key of the WireGuard interface of the CVM. string client_public_key = 1; + // Whether this CVM asked for its traffic to be gated on application health, + // i.e. `requirements.health_check` in its app-compose. When true the + // gateway polls `Health` on the v1 external surface (`/prpc/v1/Health`) and + // holds the instance out of rotation until the first poll answers. + // + // Declared here rather than discovered by probing because the poller treats + // every failed poll alike: a timeout, a refused connection, and the 404 an + // image without `/prpc/v1` answers all count as one failure, and telling an + // old image apart from a broken app would mean classifying error shapes on + // every poll -- guessing wrong either blackholes healthy legacy instances + // or lets broken ones serve. False -- including "absent", from a CVM that + // predates this field or an app that never opted in -- means the gateway + // never polls and always treats the instance as healthy, exactly as before. + bool health_check = 3; // Per-port policy the gateway should apply when proxying to this CVM. // Wrapped in a message so we can distinguish "not reported" (old CVM → // gateway falls back to fetching app-compose via Info()) from "reported @@ -104,10 +118,48 @@ message StatusResponse { repeated GatewayNodeInfo nodes = 6; // Peer uuid bytes uuid = 7; + // What the data path is actually doing with kTLS and splice. + ProxyAccelStatus accel = 8; + // Whether this node polls application health at all + // (`core.proxy.health_check.enabled`). + // + // Reported for the same reason as ProxyAccelStatus: the configured value is + // not otherwise observable at runtime, and here it changes what a host's + // `health` means. With polling off, an instance that opted in sits at + // "unknown" forever *and is fully eligible for traffic* -- indistinguishable, + // from the host list alone, from one that is being held out of rotation + // pending its first answer. That is the reading an operator is most likely to + // need during the incident that made them switch polling off. + bool health_gating = 9; +} + +// Effective state of the data-path acceleration options. +// +// The configured value does not say what is running: the startup probe can turn +// kTLS off on a kernel without the TLS ULP, and both options engage per +// connection only once their gate fires. These are reported so an operator can +// tell the difference without reading the startup log. +message ProxyAccelStatus { + // Effective kTLS mode: "off", "immediate", "after 64 KiB or 5s", or + // "disabled (kernel has no TLS ULP)" when the startup probe cleared it. + string ktls_mode = 1; + // Effective splice mode, same encoding minus the probe case. + string splice_mode = 2; + // Connections handed to the kernel's TLS ULP since start. + uint64 ktls_offloaded = 3; + // Connections whose handover to the kernel failed. + uint64 ktls_offload_failed = 4; + // Connections that entered a zero-copy splice relay, on either path. + uint64 splice_engaged = 5; } // HostInfo is the information of a host. message HostInfo { + // Field 5 was `uint32 port`, removed by b325c0f8bf when multi-port listening + // replaced the single-port field, with no reserved marker left behind. + // Reserved now so the next person to extend this message cannot silently + // reuse it and break every older client that still sends it. + reserved 5; // The Instance id string instance_id = 1; // The IP address of the host. @@ -120,6 +172,16 @@ message HostInfo { uint64 latest_handshake = 6; // The number of connections of the host. uint64 num_connections = 7; + // Whether the operator has left this instance in its app's load-balancing + // rotation (see Admin.SetInstanceReady). Absent when talking to a gateway + // that predates the traffic gate; absent must not be read as "gated off". + optional bool ready = 8; + // This node's last application-health observation: "healthy", "unhealthy", + // "unknown" (opted in, but no poll has answered yet) or "ungated" (never + // opted in, so never polled). A string rather than an enum so it stays + // readable as JSON. Empty when talking to a gateway that predates health + // polling. + string health = 9; } message QuotedPublicKey { @@ -142,6 +204,23 @@ message AcmeInfoResponse { string account_attestation = 5; } +// Result of replacing the shared ACME account credentials. +message RotateAcmeCredentialsResponse { + // URI of the newly-created ACME account. The private credentials are never returned. + string account_uri = 1; + // Number of ZT domains whose CAA records were updated for the new account. + uint32 domains_updated = 2; + // Zone-file lines an operator must publish by hand, for domains the gateway + // cannot write (dns-persist-01). Non-empty means issuance for those domains + // stays broken until they name the new account. + repeated string required_dns_records = 3; + // Domains whose CAA records could not be re-pinned to the new account. The + // rotation itself succeeded -- the account exists and the cluster is using + // it -- so this is reported rather than raised: repeating the rotation would + // register yet another account. Rerun SetCaa until it succeeds. + repeated string repin_failed_domains = 4; +} + // Get HostInfo for associated instance id. message GetInfoRequest { string id = 1; @@ -309,9 +388,15 @@ message PeerSyncStatus { uint32 id = 1; uint64 local_ack = 2; uint64 peer_ack = 3; - uint64 buffered_logs = 4; // Last seen timestamps: [(observer_node_id, timestamp), ...] - repeated LastSeenEntry last_seen = 5; + repeated LastSeenEntry last_seen = 4; + // Whether this peer has ever reported an ack map. + bool heard_from = 5; + // Consecutive quiescent rounds whose state digests disagreed. Non-zero means the + // replicas have silently diverged; wavekv 1.x could not detect this at all. + uint32 digest_mismatches = 6; + // Consecutive sync rounds that failed outright. + uint32 consecutive_failures = 7; } message LastSeenEntry { @@ -328,6 +413,13 @@ message StoreSyncStatus { bool dirty = 5; bool wal_enabled = 6; repeated PeerSyncStatus peers = 7; + // Hex SHA-256 over the replicated state. Two converged replicas produce equal + // digests by construction, so comparing this across the cluster is the promotion + // gate for the wavekv v2 rollout and the standing divergence check afterwards. + string digest = 8; + uint64 entries_merged = 9; + // Entries refused by the admission policy or the ingest quotas. + uint64 entries_rejected = 10; } // WaveKV sync status response @@ -372,13 +464,18 @@ message GetNodeStatusesResponse { repeated NodeStatusEntry statuses = 1; } +message ExitRequest { + // Exit immediately without waiting for in-flight requests to drain. + bool force = 1; +} + service Admin { // Get the status of the gateway. rpc Status(google.protobuf.Empty) returns (StatusResponse) {} // Find Proxied HostInfo by instance ID rpc GetInfo(GetInfoRequest) returns (GetInfoResponse) {} // Exit the Gateway process. - rpc Exit(google.protobuf.Empty) returns (google.protobuf.Empty) {} + rpc Exit(ExitRequest) returns (google.protobuf.Empty) {} // Renew the proxy TLS certificate if certbot is enabled rpc RenewCert(google.protobuf.Empty) returns (RenewCertResponse) {} // Reload the proxy TLS certificate from files @@ -387,7 +484,12 @@ service Admin { rpc SetCaa(google.protobuf.Empty) returns (google.protobuf.Empty) {} // Summary API for inspect. rpc GetMeta(google.protobuf.Empty) returns (GetMetaResponse) {} - // Set a node's sync URL - used for dynamic peer management + // Set a node's sync URL - used for dynamic peer management. Writing a + // node's address is also the explicit re-admission decision: it clears the + // durable removal marker RemoveNode leaves, so a node removed by mistake is + // brought back with this call. Re-admit a node with its old data directory + // only if it was never absent across a collected deletion; when in doubt, + // wipe the data directory first. rpc SetNodeUrl(SetNodeUrlRequest) returns (google.protobuf.Empty) {} // Set a node's status (up/down) rpc SetNodeStatus(SetNodeStatusRequest) returns (google.protobuf.Empty) {} @@ -399,6 +501,23 @@ service Admin { rpc GetGlobalConnections(google.protobuf.Empty) returns (GlobalConnectionsStats) {} // Get all node statuses rpc GetNodeStatuses(google.protobuf.Empty) returns (GetNodeStatusesResponse) {} + // Remove a CVM from WaveKV and the local data plane. This is an idempotent + // operator recovery action and also works when the stored record is unreadable. + rpc RemoveCvm(RemoveCvmRequest) returns (RemoveCvmResponse) {} + // List the instance records this node currently refuses to import, with the + // reason for each. Pairs with RemoveCvm: list the bad records, then remove + // the ones that should not survive. + rpc ListRejectedInstances(google.protobuf.Empty) returns (ListRejectedInstancesResponse) {} + // Remove a decommissioned gateway node from WaveKV and this node's sync peer + // set. Idempotent operator recovery action; other gateways prune the node + // from their own peer sets when the removal replicates to them. Also writes + // a durable removal marker that every gateway's sync endpoints enforce: the + // removed node's envelopes are refused, so it cannot re-register itself by + // restarting, and a stale copy of its data directory cannot resurrect + // records whose deletes the cluster has already collected. Removal is meant + // to be permanent; the escape hatch for a mistake is SetNodeUrl, which + // clears the marker. + rpc RemoveNode(RemoveNodeRequest) returns (RemoveNodeResponse) {} // ==================== DNS Credential Management ==================== // List all DNS credentials @@ -439,6 +558,23 @@ service Admin { rpc GetCertbotConfig(google.protobuf.Empty) returns (CertbotConfigResponse) {} // Set global certbot configuration (includes ACME URL) rpc SetCertbotConfig(SetCertbotConfigRequest) returns (google.protobuf.Empty) {} + // Create a new ACME account, publish the shared credentials, and re-pin + // every ZT-domain CAA record to the new account. If CAA re-pinning fails for + // some domains, the new credentials are already published; rerun SetCaa + // until it succeeds instead of retrying the rotation (each rotation + // registers a new rate-limited ACME account). Rotation is serialized across + // nodes by a best-effort lock (WaveKV has no compare-and-swap), so still + // avoid rotating from multiple gateways concurrently. This re-pins issuance + // to the new account; it does not deactivate the old ACME account at the CA. + rpc RotateAcmeCredentials(google.protobuf.Empty) returns (RotateAcmeCredentialsResponse) {} + + // ==================== Tombstone GC Configuration ==================== + // Get the effective tombstone GC pacing on this node + rpc GetTombstoneGcConfig(google.protobuf.Empty) returns (TombstoneGcConfigResponse) {} + // Store the tombstone GC pacing in the replicated KV, overriding every + // node's config-file default so the whole cluster collects on the same + // write-count boundaries. Zero disables collection cluster-wide. + rpc SetTombstoneGcConfig(SetTombstoneGcConfigRequest) returns (google.protobuf.Empty) {} // ==================== Per-Instance Port Policy Override ==================== // Set an admin override for an instance's port policy. Takes precedence @@ -451,6 +587,72 @@ service Admin { // Inspect both the admin override and the instance-reported policy for an // instance, plus the effective policy the proxy will enforce. rpc GetInstancePortPolicy(GetInstancePortPolicyRequest) returns (GetInstancePortPolicyResponse) {} + + // ==================== Per-Instance Traffic Gate ==================== + // Take an instance in or out of its app's load-balancing rotation without + // stopping the CVM. Errors if the instance is not registered. + // + // Only affects connections addressed to the app id. Connections addressed + // to the instance id are never gated, so an instance pulled from rotation + // stays directly reachable for investigation. + // + // The setting is persisted under a key only the Admin API writes, so a + // re-registration -- by this instance or from another gateway node -- cannot + // reopen the gate. An instance does not quietly rejoin the rotation by + // rebooting, and does not need the operator to re-issue the gate after one. + rpc SetInstanceReady(SetInstanceReadyRequest) returns (google.protobuf.Empty) {} +} + +// Emergency operator request to remove one CVM's instance record. +message RemoveCvmRequest { + string instance_id = 1; +} + +// Outcome of a RemoveCvm request. Both fields are false when the request +// names an instance this cluster has never seen (or a retry of a removal +// that already completed), so a mistyped instance_id is visible to the +// operator instead of silently reporting success. +message RemoveCvmResponse { + // Whether a live instance record existed in WaveKV before the tombstone + // was written. Also true for records that existed but were unreadable. + bool record_existed = 1; + // Whether the CVM was present in this node's local data plane. + bool removed_locally = 2; +} + +// One instance record this node refuses to import. +message RejectedInstanceInfo { + string instance_id = 1; + // Why the record is refused. + string reason = 2; + // "unusable": the record fails validation or its bytes no longer decode. + // "lost_conflict": the record lost an IP or key conflict to an older + // registration. + string rejection = 3; + // Whether the instance still holds state in this node's data plane. An + // unusable record keeps whatever the data plane already had, so removing + // an active instance also drops its routing. + bool active_locally = 4; +} + +message ListRejectedInstancesResponse { + repeated RejectedInstanceInfo rejected = 1; +} + +// Emergency operator request to remove a decommissioned gateway node. +message RemoveNodeRequest { + uint32 node_id = 1; +} + +// Outcome of RemoveNode. Both fields are false when the node was never known +// (or the removal already completed), so a mistyped node_id is visible to +// the operator instead of silently reporting success. +message RemoveNodeResponse { + // Whether any of the node's records (info, status, or sync address) was + // live in WaveKV before the tombstones were written. + bool record_existed = 1; + // Whether the node was still in this gateway's sync peer set. + bool removed_from_peer_set = 2; } // ==================== DNS Credential Messages ==================== @@ -549,6 +751,21 @@ message ZtDomainConfig { optional uint32 node = 4; // Priority for default base_domain selection (higher = preferred) int32 priority = 5; + // ACME challenge proving control of this domain: "dns-01" (default) or + // "dns-persist-01". Empty means "dns-01". + // + // "dns-persist-01" needs no DNS credential: control comes from a + // _validation-persist TXT record the operator publishes once, so the gateway + // CVM never holds write access to the zone. Experimental — the draft is still + // changing and Let's Encrypt serves it on staging only. + // + // Optional so that omitting it means "leave it alone" rather than "dns-01". + // UpdateZtDomain replaces the whole record, so a caller that predates this + // field -- a cached dashboard bundle, a script, an older SDK -- would + // otherwise downgrade a dns-persist-01 domain to dns-01 cluster-wide on any + // unrelated edit, and its hand-published CAA naming dns-persist-01 then + // refuses every order. + optional string challenge = 6; } // ZT-Domain information (config + certificate status) @@ -557,6 +774,10 @@ message ZtDomainInfo { ZtDomainConfig config = 1; // Certificate status ZtDomainCertStatus cert_status = 2; + // Zone-file lines the domain's DNS must contain. Under "dns-01" the gateway + // writes these itself and they are informational; under "dns-persist-01" they + // are the one-time setup an operator has to publish. + repeated string required_dns_records = 3; } // ZT-Domain certificate status @@ -612,12 +833,16 @@ message ForceReleaseCertLockRequest { message CertAttestationInfo { // Certificate public key (DER encoded) bytes public_key = 1; - // TDX Quote (JSON serialized) + // TDX Quote (JSON serialized). Empty when the node has no Intel TDX, or when + // the quote request failed; `attestation` is the evidence then. string quote = 2; // Node that generated this attestation uint32 generated_by = 3; // Timestamp when this attestation was generated uint64 generated_at = 4; + // Versioned attestation (JSON serialized `AttestResponse`). Present on every + // platform, and the only evidence on nodes without Intel TDX. + string attestation = 5; } // List certificate attestations request @@ -647,6 +872,9 @@ message CertbotConfigResponse { uint64 renew_timeout_secs = 3; // ACME server URL (empty means default Let's Encrypt production) string acme_url = 4; + // Issuer Domain Name naming the CA in dns-persist-01 and CAA records + // (empty means letsencrypt.org) + string issuer_domain_name = 5; } // Set certbot configuration request @@ -659,6 +887,27 @@ message SetCertbotConfigRequest { optional uint64 renew_timeout_secs = 3; // ACME server URL (empty means use default Let's Encrypt production) optional string acme_url = 4; + // Issuer Domain Name naming the CA in dns-persist-01 and CAA records + // (empty means letsencrypt.org). Must be one of the issuer-domain-names the + // ACME server at acme_url advertises, or every dns-persist-01 order fails. + optional string issuer_domain_name = 5; +} + +// ==================== Tombstone GC Configuration Messages ==================== + +// Tombstone GC configuration response +message TombstoneGcConfigResponse { + // Collect tombstones once every this many replicated writes (0 = disabled) + uint64 writes_per_collection = 1; + // True when the value is the replicated KV override; false when it is this + // node's config-file default (which other nodes may not share) + bool stored_in_kv = 2; +} + +// Set tombstone GC configuration request +message SetTombstoneGcConfigRequest { + // Collect tombstones once every this many replicated writes (0 = disabled) + uint64 writes_per_collection = 1; } // ==================== Per-Instance Port Policy Override Messages ==================== @@ -677,6 +926,22 @@ message ClearInstancePortPolicyRequest { string instance_id = 1; } +// Take an instance in or out of its app's load-balancing rotation. +message SetInstanceReadyRequest { + // The instance to gate. + string instance_id = 1; + // true puts the instance back into rotation, false removes it. Existing + // connections are left running either way; this only affects which + // instances new connections may be routed to. + // + // Required. It is `optional` so that absent and false stay distinguishable: + // a proto3 scalar decodes an omitted field as its zero value, and over the + // JSON transport -- where unknown keys are ignored -- a request that omits + // the field or misspells it would otherwise be a successful call that pulls + // the instance out of rotation. The server rejects absent instead. + optional bool ready = 2; +} + // Inspect an instance's port-policy state. message GetInstancePortPolicyRequest { string instance_id = 1; diff --git a/dstack/gateway/src/admin_auth.rs b/dstack/gateway/src/admin_auth.rs index 27fa3025d..06c9f0b62 100644 --- a/dstack/gateway/src/admin_auth.rs +++ b/dstack/gateway/src/admin_auth.rs @@ -1,558 +1,70 @@ -// SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> +// SPDX-FileCopyrightText: © 2025-2026 Phala Network <dstack@phala.network> // // SPDX-License-Identifier: Apache-2.0 -//! Admin server authentication. -//! -//! Attaches to the admin Rocket instance and rejects requests that do not -//! present the configured shared secret. The token is accepted via, in order: -//! 1. `X-Admin-Token` header (any method) -//! 2. `Authorization: Bearer <token>` header (any method) -//! 3. `Authorization: Basic <base64(user:token)>` (any method; token may be -//! in either the user or password field — needed so plain browsers can -//! authenticate to the dashboard via the native HTTP-auth prompt) -//! 4. `?token=<token>` query parameter (GET only, for dashboard links) -//! -//! For (4), the `token` query parameter is stripped from the request URI after -//! successful validation so it doesn't propagate to access logs, downstream -//! handlers, or the Referer header. -//! -//! Rejected requests are forwarded to a sentinel route that returns HTTP 401 -//! with `WWW-Authenticate: Basic realm="dstack-gateway admin"` so browsers -//! show the native login prompt. All admin routes (prpc-generated and -//! dashboard) are protected by this single fairing attachment without -//! modifying the route declarations. -//! -//! The token is only ever held in memory as its SHA-256 hash; the configured -//! plaintext is dropped right after the fairing is constructed. +//! Backwards-compatible Gateway adapter for the shared API authenticator. use anyhow::{bail, Result}; -use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; -use rocket::{ - fairing::{Fairing, Info, Kind}, - http::{uri::Origin, Header, Method, Status}, - response::Responder, - Data, Request, Response, Route, -}; -use sha2::{Digest, Sha256}; -use subtle::ConstantTimeEq; +use dstack_api_auth::{Authenticator, HttpAuthConfig, HttpAuthFairing}; +use rocket::Route; use crate::config::AdminConfig; -const UNAUTH_URI: &str = "/__admin_unauthorized"; -const HEADER_NAME: &str = "X-Admin-Token"; -const QUERY_PARAM: &str = "token"; const ENV_ADMIN_TOKEN: &str = "DSTACK_GATEWAY_ADMIN_TOKEN"; const ENV_ADMIN_TOKEN_COMPAT: &str = "ADMIN_API_TOKEN"; -const BASIC_REALM: &str = "dstack-gateway admin"; -pub struct AdminAuthFairing { - /// SHA-256 of the configured token. `None` = auth disabled (insecure mode). - token_hash: Option<[u8; 32]>, -} +pub struct AdminAuthFairing(HttpAuthFairing); impl AdminAuthFairing { - /// Build a fairing from a resolved plaintext token. Empty disables auth. - pub fn new(token: &str) -> Self { - Self { - token_hash: (!token.is_empty()).then(|| sha256(token.as_bytes())), - } - } - - /// Resolve a token from config + env, applying the auth policy: - /// - `insecure_no_auth = true` → disabled (caller is expected to warn) - /// - else require a non-empty token from `admin_token`, - /// `DSTACK_GATEWAY_ADMIN_TOKEN`, or `ADMIN_API_TOKEN`. pub fn from_config(config: &AdminConfig) -> Result<Self> { if config.insecure_no_auth { - return Ok(Self { token_hash: None }); + return Ok(Self(HttpAuthFairing::new( + Authenticator::disabled(), + http_config(), + ))); } - let token = if !config.admin_token.is_empty() { - config.admin_token.clone() + let token = if !config.auth_token.is_empty() { + config.auth_token.trim().to_owned() } else { std::env::var(ENV_ADMIN_TOKEN) .or_else(|_| std::env::var(ENV_ADMIN_TOKEN_COMPAT)) .unwrap_or_default() + .trim() + .to_owned() }; - let token = token.trim(); - if token.is_empty() { + if token.is_empty() && config.htpasswd_file.as_os_str().is_empty() { bail!( - "admin API is enabled but no admin_token is configured; \ - set core.admin.admin_token, {ENV_ADMIN_TOKEN}, or {ENV_ADMIN_TOKEN_COMPAT}, \ - or set core.admin.insecure_no_auth = true (testing only)" + "admin API is enabled but neither auth_token nor htpasswd_file is configured; \ + set core.admin.auth_token, {ENV_ADMIN_TOKEN}, {ENV_ADMIN_TOKEN_COMPAT}, \ + core.admin.htpasswd_file, or insecure_no_auth = true (testing only)" ); } - Ok(Self::new(token)) - } - - fn extract_token(req: &Request<'_>) -> Option<String> { - if let Some(t) = req.headers().get_one(HEADER_NAME) { - return Some(t.to_string()); - } - if let Some(auth) = req.headers().get_one("Authorization") { - if let Some(t) = auth.strip_prefix("Bearer ") { - return Some(t.trim().to_string()); - } - if let Some(b64) = auth.strip_prefix("Basic ") { - if let Some(t) = basic_auth_token(b64.trim()) { - return Some(t); - } - } + let mut auth = Authenticator::from_tokens([token]); + if !config.htpasswd_file.as_os_str().is_empty() { + auth = auth.with_htpasswd_file(&config.htpasswd_file)?; } - // Query token is intended for browser links to the dashboard, so only - // accept it on GET to avoid leaking via mutating request URIs. - if req.method() == Method::Get { - for field in req.query_fields() { - if field.name.key_lossy().as_str() == QUERY_PARAM { - return Some(field.value.to_string()); - } - } - } - None - } -} - -fn sha256(bytes: &[u8]) -> [u8; 32] { - Sha256::digest(bytes).into() -} - -/// Decode a `Basic` credential and return whichever of user/password is -/// non-empty (we accept either so the browser prompt's two fields are -/// interchangeable for the operator). -fn basic_auth_token(b64: &str) -> Option<String> { - let decoded = BASE64.decode(b64).ok()?; - let text = std::str::from_utf8(&decoded).ok()?; - let (user, pass) = text.split_once(':').unwrap_or((text, "")); - if !pass.is_empty() { - return Some(pass.to_string()); - } - if !user.is_empty() { - return Some(user.to_string()); + Ok(Self(HttpAuthFairing::new(auth, http_config()))) } - None } -/// 401 response that triggers the browser's native HTTP-auth prompt. -struct Unauthorized; - -impl<'r> Responder<'r, 'static> for Unauthorized { - fn respond_to(self, _req: &'r Request<'_>) -> rocket::response::Result<'static> { - Response::build() - .status(Status::Unauthorized) - .header(Header::new( - "WWW-Authenticate", - format!("Basic realm=\"{BASIC_REALM}\""), - )) - .ok() +fn http_config() -> HttpAuthConfig { + HttpAuthConfig { + realm: "dstack-gateway admin".into(), + token_header: Some("X-Admin-Token".into()), + allow_get_query_token: true, } } -/// Rebuild the request URI without the `token` query parameter, if present. -/// Returns `None` when there is nothing to strip. -fn strip_token_query(uri: &Origin<'_>) -> Option<Origin<'static>> { - let query = uri.query()?.as_str(); - let mut kept = Vec::new(); - let mut found = false; - for pair in query.split('&') { - let key = pair.split('=').next().unwrap_or(""); - if key == QUERY_PARAM { - found = true; - } else if !pair.is_empty() { - kept.push(pair); - } - } - if !found { - return None; - } - let path = uri.path().as_str(); - let new_uri = if kept.is_empty() { - path.to_string() - } else { - format!("{}?{}", path, kept.join("&")) - }; - Origin::parse_owned(new_uri).ok() -} - #[rocket::async_trait] -impl Fairing for AdminAuthFairing { - fn info(&self) -> Info { - Info { - name: "admin auth", - kind: Kind::Request, - } +impl rocket::fairing::Fairing for AdminAuthFairing { + fn info(&self) -> rocket::fairing::Info { + self.0.info() } - - async fn on_request(&self, req: &mut Request<'_>, _: &mut Data<'_>) { - let Some(expected_hash) = self.token_hash.as_ref() else { - return; - }; - // Avoid infinite re-routing if the fairing fires on the sentinel itself. - if req.uri().path() == UNAUTH_URI { - return; - } - let provided = Self::extract_token(req).unwrap_or_default(); - let provided_hash = sha256(provided.as_bytes()); - let matches: bool = provided_hash.ct_eq(expected_hash).into(); - if !matches { - if let Ok(origin) = Origin::parse_owned(UNAUTH_URI.to_string()) { - req.set_uri(origin); - } - return; - } - // Authorized — strip ?token=... so it doesn't propagate to logs/handlers. - if let Some(stripped) = strip_token_query(req.uri()) { - req.set_uri(stripped); - } + async fn on_request(&self, req: &mut rocket::Request<'_>, data: &mut rocket::Data<'_>) { + self.0.on_request(req, data).await } } -// Sentinel 401 handlers for every HTTP method Rocket can dispatch. We have to -// enumerate them because Rocket doesn't support a method-agnostic route. - -#[rocket::get("/__admin_unauthorized")] -fn unauth_get() -> Unauthorized { - Unauthorized -} - -#[rocket::post("/__admin_unauthorized", data = "<_data>")] -fn unauth_post(_data: Data<'_>) -> Unauthorized { - Unauthorized -} - -#[rocket::put("/__admin_unauthorized", data = "<_data>")] -fn unauth_put(_data: Data<'_>) -> Unauthorized { - Unauthorized -} - -#[rocket::patch("/__admin_unauthorized", data = "<_data>")] -fn unauth_patch(_data: Data<'_>) -> Unauthorized { - Unauthorized -} - -#[rocket::delete("/__admin_unauthorized")] -fn unauth_delete() -> Unauthorized { - Unauthorized -} - -#[rocket::options("/__admin_unauthorized")] -fn unauth_options() -> Unauthorized { - Unauthorized -} - -#[rocket::head("/__admin_unauthorized")] -fn unauth_head() -> Unauthorized { - Unauthorized -} - pub fn routes() -> Vec<Route> { - rocket::routes![ - unauth_get, - unauth_post, - unauth_put, - unauth_patch, - unauth_delete, - unauth_options, - unauth_head, - ] -} - -#[cfg(test)] -mod tests { - use super::*; - use rocket::http::{ContentType, Header, Status}; - use rocket::local::asynchronous::Client; - - #[rocket::get("/protected")] - fn protected_get() -> &'static str { - "ok" - } - - #[rocket::post("/protected", data = "<_data>")] - fn protected_post(_data: Data<'_>) -> &'static str { - "ok" - } - - #[rocket::get("/echo?<token>&<other>")] - fn echo(token: Option<&str>, other: Option<&str>) -> String { - format!( - "token={} other={}", - token.unwrap_or("<absent>"), - other.unwrap_or("<absent>") - ) - } - - async fn make_client(token: &str) -> Client { - let r = rocket::build() - .attach(AdminAuthFairing::new(token)) - .mount("/", routes()) - .mount("/", rocket::routes![protected_get, protected_post, echo]); - Client::tracked(r).await.unwrap() - } - - #[rocket::async_test] - async fn empty_token_disables_auth() { - let client = make_client("").await; - let resp = client.get("/protected").dispatch().await; - assert_eq!(resp.status(), Status::Ok); - let resp = client.post("/protected").dispatch().await; - assert_eq!(resp.status(), Status::Ok); - } - - #[rocket::async_test] - async fn missing_token_returns_401() { - let client = make_client("s3cret").await; - let resp = client.get("/protected").dispatch().await; - assert_eq!(resp.status(), Status::Unauthorized); - let resp = client.post("/protected").dispatch().await; - assert_eq!(resp.status(), Status::Unauthorized); - } - - #[rocket::async_test] - async fn header_token_accepted() { - let client = make_client("s3cret").await; - let resp = client - .get("/protected") - .header(Header::new(HEADER_NAME, "s3cret")) - .dispatch() - .await; - assert_eq!(resp.status(), Status::Ok); - let resp = client - .post("/protected") - .header(ContentType::JSON) - .header(Header::new(HEADER_NAME, "s3cret")) - .dispatch() - .await; - assert_eq!(resp.status(), Status::Ok); - } - - #[rocket::async_test] - async fn bearer_token_accepted() { - let client = make_client("s3cret").await; - let resp = client - .get("/protected") - .header(Header::new("Authorization", "Bearer s3cret")) - .dispatch() - .await; - assert_eq!(resp.status(), Status::Ok); - } - - #[rocket::async_test] - async fn wrong_token_rejected() { - let client = make_client("s3cret").await; - let resp = client - .get("/protected") - .header(Header::new(HEADER_NAME, "wrong")) - .dispatch() - .await; - assert_eq!(resp.status(), Status::Unauthorized); - } - - #[rocket::async_test] - async fn header_takes_precedence_over_query() { - let client = make_client("s3cret").await; - // Wrong query token but correct header → authorized. - let resp = client - .get("/protected?token=wrong") - .header(Header::new(HEADER_NAME, "s3cret")) - .dispatch() - .await; - assert_eq!(resp.status(), Status::Ok); - } - - #[rocket::async_test] - async fn query_token_only_accepted_on_get() { - let client = make_client("s3cret").await; - // GET with ?token= → allowed - let resp = client.get("/protected?token=s3cret").dispatch().await; - assert_eq!(resp.status(), Status::Ok); - // POST with ?token= → rejected (query auth not honored on mutating methods) - let resp = client.post("/protected?token=s3cret").dispatch().await; - assert_eq!(resp.status(), Status::Unauthorized); - } - - #[rocket::async_test] - async fn query_token_stripped_after_auth() { - let client = make_client("s3cret").await; - // Token is stripped → handler sees no `token` param, only `other`. - let resp = client.get("/echo?token=s3cret&other=keep").dispatch().await; - assert_eq!(resp.status(), Status::Ok); - let body = resp.into_string().await.unwrap(); - assert_eq!(body, "token=<absent> other=keep"); - } - - #[rocket::async_test] - async fn query_token_stripped_when_authed_via_header() { - let client = make_client("s3cret").await; - let resp = client - .get("/echo?token=anything&other=keep") - .header(Header::new(HEADER_NAME, "s3cret")) - .dispatch() - .await; - assert_eq!(resp.status(), Status::Ok); - let body = resp.into_string().await.unwrap(); - assert_eq!(body, "token=<absent> other=keep"); - } - - fn hash_of(fairing: &AdminAuthFairing) -> Option<[u8; 32]> { - fairing.token_hash - } - - #[test] - fn from_config_disabled_when_insecure_flag_set() { - let cfg = AdminConfig { - enabled: true, - admin_token: String::new(), - insecure_no_auth: true, - }; - let fairing = match AdminAuthFairing::from_config(&cfg) { - Ok(f) => f, - Err(e) => panic!("expected Ok, got err: {e}"), - }; - assert!(hash_of(&fairing).is_none()); - } - - #[test] - fn from_config_uses_config_token() { - let cfg = AdminConfig { - enabled: true, - admin_token: "from-config".into(), - insecure_no_auth: false, - }; - let fairing = match AdminAuthFairing::from_config(&cfg) { - Ok(f) => f, - Err(e) => panic!("expected Ok, got err: {e}"), - }; - assert_eq!(hash_of(&fairing), Some(sha256(b"from-config"))); - } - - // Env-touching cases are combined into a single test so cargo's parallel - // runner doesn't race on `DSTACK_GATEWAY_ADMIN_TOKEN` / `ADMIN_API_TOKEN`. - #[test] - fn from_config_env_paths() { - let empty_cfg = AdminConfig { - enabled: true, - admin_token: String::new(), - insecure_no_auth: false, - }; - - // Baseline: no env, no config token → error. - unsafe { - std::env::remove_var(ENV_ADMIN_TOKEN); - std::env::remove_var(ENV_ADMIN_TOKEN_COMPAT); - } - let err = match AdminAuthFairing::from_config(&empty_cfg) { - Err(e) => e, - Ok(_) => panic!("expected error, got Ok"), - }; - assert!(err.to_string().contains("no admin_token is configured")); - - // Primary env var picked up. - unsafe { - std::env::set_var(ENV_ADMIN_TOKEN, "from-env"); - } - let fairing = match AdminAuthFairing::from_config(&empty_cfg) { - Ok(f) => f, - Err(e) => panic!("expected Ok, got err: {e}"), - }; - assert_eq!(hash_of(&fairing), Some(sha256(b"from-env"))); - unsafe { - std::env::remove_var(ENV_ADMIN_TOKEN); - } - - // Compat env var picked up when primary is absent. - unsafe { - std::env::set_var(ENV_ADMIN_TOKEN_COMPAT, "from-compat"); - } - let fairing = match AdminAuthFairing::from_config(&empty_cfg) { - Ok(f) => f, - Err(e) => panic!("expected Ok, got err: {e}"), - }; - assert_eq!(hash_of(&fairing), Some(sha256(b"from-compat"))); - unsafe { - std::env::remove_var(ENV_ADMIN_TOKEN_COMPAT); - } - } - - #[rocket::async_test] - async fn unauth_returns_401_on_all_methods() { - let client = make_client("s3cret").await; - // PUT / DELETE / PATCH / OPTIONS to a protected URI with no token - // should be rewritten to the sentinel and return 401, not 404. - for m in [Method::Put, Method::Delete, Method::Patch, Method::Options] { - let resp = client.req(m, "/protected").dispatch().await; - assert_eq!( - resp.status(), - Status::Unauthorized, - "method {m:?} expected 401, got {}", - resp.status() - ); - } - } - - fn basic_header(user: &str, pass: &str) -> Header<'static> { - let creds = format!("{user}:{pass}"); - Header::new("Authorization", format!("Basic {}", BASE64.encode(creds))) - } - - #[rocket::async_test] - async fn basic_auth_password_field_accepted() { - let client = make_client("s3cret").await; - let resp = client - .get("/protected") - .header(basic_header("admin", "s3cret")) - .dispatch() - .await; - assert_eq!(resp.status(), Status::Ok); - } - - #[rocket::async_test] - async fn basic_auth_user_field_accepted_when_password_empty() { - let client = make_client("s3cret").await; - // Some browser users paste the token into the username field by mistake. - let resp = client - .get("/protected") - .header(basic_header("s3cret", "")) - .dispatch() - .await; - assert_eq!(resp.status(), Status::Ok); - } - - #[rocket::async_test] - async fn basic_auth_wrong_password_rejected() { - let client = make_client("s3cret").await; - let resp = client - .get("/protected") - .header(basic_header("admin", "wrong")) - .dispatch() - .await; - assert_eq!(resp.status(), Status::Unauthorized); - } - - #[rocket::async_test] - async fn basic_auth_malformed_rejected() { - let client = make_client("s3cret").await; - // Not valid base64 at all. - let resp = client - .get("/protected") - .header(Header::new("Authorization", "Basic !!not-base64!!")) - .dispatch() - .await; - assert_eq!(resp.status(), Status::Unauthorized); - } - - #[rocket::async_test] - async fn unauthorized_response_includes_www_authenticate() { - let client = make_client("s3cret").await; - let resp = client.get("/protected").dispatch().await; - assert_eq!(resp.status(), Status::Unauthorized); - let www = resp - .headers() - .get_one("WWW-Authenticate") - .expect("missing WWW-Authenticate header"); - assert!( - www.starts_with("Basic realm="), - "expected Basic challenge, got {www:?}" - ); - assert!(www.contains("dstack-gateway admin")); - } + dstack_api_auth::routes() } diff --git a/dstack/gateway/src/admin_service.rs b/dstack/gateway/src/admin_service.rs index 8a02bdb19..d840642e3 100644 --- a/dstack/gateway/src/admin_service.rs +++ b/dstack/gateway/src/admin_service.rs @@ -5,33 +5,40 @@ use std::sync::atomic::Ordering; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use anyhow::{bail, Context, Result}; +use anyhow::{bail, ensure, Context, Result}; +use certbot::ChallengeKind; use dstack_gateway_rpc::{ admin_server::{AdminRpc, AdminServer}, CertAttestationInfo, CertbotConfigResponse, ClearInstancePortPolicyRequest, CreateDnsCredentialRequest, DeleteDnsCredentialRequest, DeleteZtDomainRequest, - DnsCredentialInfo, ForceReleaseCertLockRequest, GetDefaultDnsCredentialResponse, + DnsCredentialInfo, ExitRequest, ForceReleaseCertLockRequest, GetDefaultDnsCredentialResponse, GetDnsCredentialRequest, GetInfoRequest, GetInfoResponse, GetInstanceHandshakesRequest, GetInstanceHandshakesResponse, GetInstancePortPolicyRequest, GetInstancePortPolicyResponse, GetMetaResponse, GetNodeStatusesResponse, GetZtDomainRequest, GlobalConnectionsStats, HandshakeEntry, HostInfo, LastSeenEntry, ListCertAttestationsRequest, - ListCertAttestationsResponse, ListDnsCredentialsResponse, ListZtDomainsResponse, - NodeStatusEntry, PeerSyncStatus as ProtoPeerSyncStatus, PortAttrs as RpcPortAttrs, - PortPolicy as RpcPortPolicy, RenewCertResponse, RenewZtDomainCertRequest, - RenewZtDomainCertResponse, SetCertbotConfigRequest, SetDefaultDnsCredentialRequest, - SetInstancePortPolicyRequest, SetNodeStatusRequest, SetNodeUrlRequest, StatusResponse, - StoreSyncStatus, UpdateDnsCredentialRequest, WaveKvStatusResponse, ZtDomainCertStatus, - ZtDomainConfig as ProtoZtDomainConfig, ZtDomainInfo, + ListCertAttestationsResponse, ListDnsCredentialsResponse, ListRejectedInstancesResponse, + ListZtDomainsResponse, NodeStatusEntry, PeerSyncStatus as ProtoPeerSyncStatus, + PortAttrs as RpcPortAttrs, PortPolicy as RpcPortPolicy, RejectedInstanceInfo, RemoveCvmRequest, + RemoveCvmResponse, RemoveNodeRequest, RemoveNodeResponse, RenewCertResponse, + RenewZtDomainCertRequest, RenewZtDomainCertResponse, RotateAcmeCredentialsResponse, + SetCertbotConfigRequest, SetDefaultDnsCredentialRequest, SetInstancePortPolicyRequest, + SetInstanceReadyRequest, SetNodeStatusRequest, SetNodeUrlRequest, SetTombstoneGcConfigRequest, + StatusResponse, StoreSyncStatus, TombstoneGcConfigResponse, UpdateDnsCredentialRequest, + WaveKvStatusResponse, ZtDomainCertStatus, ZtDomainConfig as ProtoZtDomainConfig, ZtDomainInfo, }; use ra_rpc::{CallContext, RpcCall}; -use tracing::info; +use tracing::{info, warn}; use wavekv::node::NodeStatus as WaveKvNodeStatus; use crate::{ - kv::{DnsCredential, DnsProvider, NodeStatus, PortFlags, PortPolicy, ZtDomainConfig}, + kv::{ + import::Rejection, DnsCredential, DnsProvider, GlobalCertbotConfig, + GlobalTombstoneGcConfig, NodeStatus, PortFlags, PortPolicy, ZtDomainConfig, + }, main_service::Proxy, models::PortPolicyView, - proxy::NUM_CONNECTIONS, + proxy::{stats::accel_status, NUM_CONNECTIONS}, + time::now_secs, }; pub struct AdminRpcHandler { @@ -63,6 +70,8 @@ impl AdminRpcHandler { base_domain: base_domain.clone(), latest_handshake, num_connections: instance.num_connections(), + ready: Some(instance.is_ready()), + health: instance.health().as_str().to_string(), } }) .collect::<Vec<_>>(); @@ -74,13 +83,17 @@ impl AdminRpcHandler { nodes: state.get_all_nodes(), hosts, num_connections: NUM_CONNECTIONS.load(Ordering::Relaxed), + // Reads the post-probe config, so this is what the data path is + // running rather than what the file asked for. + accel: Some(accel_status(&state.config.proxy)), + health_gating: state.config.proxy.health_check.enabled, }) } } impl AdminRpc for AdminRpcHandler { - async fn exit(self) -> Result<()> { - self.state.lock().exit(); + async fn exit(self, request: ExitRequest) -> Result<()> { + self.state.lock().exit(request.force) } async fn renew_cert(self) -> Result<RenewCertResponse> { @@ -90,15 +103,23 @@ impl AdminRpc for AdminRpcHandler { } async fn set_caa(self) -> Result<()> { - // TODO: Implement CAA setting for multi-domain certificates - // This requires iterating over all domain configurations and setting CAA records - bail!("set_caa is not implemented for multi-domain certificates yet"); + self.state.certbot.set_caa_all().await } async fn reload_cert(self) -> Result<()> { self.state.reload_all_certs_from_kvstore() } + async fn rotate_acme_credentials(self) -> Result<RotateAcmeCredentialsResponse> { + let outcome = self.state.rotate_acme_credentials().await?; + Ok(RotateAcmeCredentialsResponse { + account_uri: outcome.account_uri, + domains_updated: outcome.domains_updated.try_into().unwrap_or(u32::MAX), + required_dns_records: outcome.required_dns_records, + repin_failed_domains: outcome.repin_failed, + }) + } + async fn status(self) -> Result<StatusResponse> { self.status().await } @@ -126,6 +147,8 @@ impl AdminRpc for AdminRpcHandler { ts }, num_connections: instance.num_connections(), + ready: Some(instance.is_ready()), + health: instance.health().as_str().to_string(), }; Ok(GetInfoResponse { found: true, @@ -169,6 +192,11 @@ impl AdminRpc for AdminRpcHandler { async fn set_node_url(self, request: SetNodeUrlRequest) -> Result<()> { let kv_store = self.state.kv_store(); + // Writing a node's address back is the explicit re-admission + // decision, so it clears the removal marker the sync lockout reads. + if kv_store.clear_peer_removed(request.id)? { + info!("cleared the removal marker for node {}", request.id); + } kv_store.register_peer_url(request.id, &request.url)?; info!("Updated peer URL: node {} -> {}", request.id, request.url); Ok(()) @@ -199,16 +227,33 @@ impl AdminRpc for AdminRpcHandler { .collect() }; + // Per-peer digest and failure telemetry lives on the sync manager, not the store. + let links = self + .state + .wavekv_sync + .as_ref() + .map(|s| s.link_status()) + .unwrap_or_default(); + let links_for = |name: &str| -> Vec<wavekv::sync::PeerLinkStatus> { + links + .iter() + .find(|(store, _)| *store == name) + .map(|(_, l)| l.clone()) + .unwrap_or_default() + }; + Ok(WaveKvStatusResponse { enabled: self.state.config.sync.enabled, persistent: Some(build_store_status( "persistent", persistent_status, + &links_for("persistent"), &get_peer_last_seen, )), ephemeral: Some(build_store_status( "ephemeral", ephemeral_status, + &links_for("ephemeral"), &get_peer_last_seen, )), }) @@ -292,6 +337,56 @@ impl AdminRpc for AdminRpcHandler { Ok(GetNodeStatusesResponse { statuses: entries }) } + async fn remove_cvm(self, request: RemoveCvmRequest) -> Result<RemoveCvmResponse> { + let instance_id = request.instance_id.as_str(); + // Same bound the KV import boundary puts on identifiers. Legitimate + // gateways never write an instance_id outside it, so this rejects only + // typos — and keeps the ID safe to embed in logs and KV keys. + crate::kv::import::validate_id("instance_id", instance_id)?; + + let removal = self.state.remove_cvm(instance_id)?; + warn!( + "admin removed CVM {instance_id} from WaveKV and the local data plane \ + (record existed: {}, present locally: {})", + removal.record_existed, removal.removed_locally + ); + Ok(RemoveCvmResponse { + record_existed: removal.record_existed, + removed_locally: removal.removed_locally, + }) + } + + async fn list_rejected_instances(self) -> Result<ListRejectedInstancesResponse> { + let rejected = self + .state + .rejected_instances() + .into_iter() + .map(|report| RejectedInstanceInfo { + instance_id: report.rejected.instance_id, + reason: format!("{:#}", report.rejected.reason), + rejection: match report.rejected.rejection { + Rejection::Unusable => "unusable".to_string(), + Rejection::LostConflict => "lost_conflict".to_string(), + }, + active_locally: report.active_locally, + }) + .collect(); + Ok(ListRejectedInstancesResponse { rejected }) + } + + async fn remove_node(self, request: RemoveNodeRequest) -> Result<RemoveNodeResponse> { + let removal = self.state.remove_node(request.node_id)?; + warn!( + "admin removed node {} from WaveKV and the sync peer set \ + (record existed: {}, was a sync peer: {})", + request.node_id, removal.record_existed, removal.removed_from_peer_set + ); + Ok(RemoveNodeResponse { + record_existed: removal.record_existed, + removed_from_peer_set: removal.removed_from_peer_set, + }) + } + // ==================== DNS Credential Management ==================== async fn list_dns_credentials(self) -> Result<ListDnsCredentialsResponse> { @@ -301,7 +396,7 @@ impl AdminRpc for AdminRpcHandler { .into_iter() .map(dns_cred_to_proto) .collect(); - let default_id = kv_store.get_default_dns_credential_id(); + let default_id = kv_store.get_default_dns_credential_id()?; Ok(ListDnsCredentialsResponse { credentials, default_id, @@ -314,7 +409,7 @@ impl AdminRpc for AdminRpcHandler { ) -> Result<DnsCredentialInfo> { let kv_store = self.state.kv_store(); let cred = kv_store - .get_dns_credential(&request.id) + .get_dns_credential(&request.id)? .context("dns credential not found")?; Ok(dns_cred_to_proto(cred)) } @@ -337,7 +432,14 @@ impl AdminRpc for AdminRpcHandler { let now = now_secs(); let id = generate_cred_id(); let dns_txt_ttl = request.dns_txt_ttl.unwrap_or(60); - let max_dns_wait = Duration::from_secs(request.max_dns_wait.unwrap_or(60 * 5).into()); + let max_dns_wait_secs = request.max_dns_wait.unwrap_or(60 * 5); + if dns_txt_ttl == 0 { + bail!("dns_txt_ttl must be greater than zero"); + } + if max_dns_wait_secs == 0 { + bail!("max_dns_wait must be greater than zero"); + } + let max_dns_wait = Duration::from_secs(max_dns_wait_secs.into()); let cred = DnsCredential { id: id.clone(), name: request.name, @@ -367,7 +469,7 @@ impl AdminRpc for AdminRpcHandler { let kv_store = self.state.kv_store(); let mut cred = kv_store - .get_dns_credential(&request.id) + .get_dns_credential(&request.id)? .context("dns credential not found")?; // Update name if provided @@ -398,7 +500,7 @@ impl AdminRpc for AdminRpcHandler { let kv_store = self.state.kv_store(); // Check if this is the default credential - if let Some(default_id) = kv_store.get_default_dns_credential_id() { + if let Some(default_id) = kv_store.get_default_dns_credential_id()? { if default_id == request.id { bail!("cannot delete the default DNS credential; set a different default first"); } @@ -422,8 +524,12 @@ impl AdminRpc for AdminRpcHandler { async fn get_default_dns_credential(self) -> Result<GetDefaultDnsCredentialResponse> { let kv_store = self.state.kv_store(); - let default_id = kv_store.get_default_dns_credential_id().unwrap_or_default(); - let credential = kv_store.get_default_dns_credential().map(dns_cred_to_proto); + let default_id = kv_store + .get_default_dns_credential_id()? + .unwrap_or_default(); + let credential = kv_store + .get_default_dns_credential()? + .map(dns_cred_to_proto); Ok(GetDefaultDnsCredentialResponse { default_id, credential, @@ -438,7 +544,7 @@ impl AdminRpc for AdminRpcHandler { // Verify the credential exists kv_store - .get_dns_credential(&request.id) + .get_dns_credential(&request.id)? .context("dns credential not found")?; kv_store.set_default_dns_credential_id(&request.id)?; @@ -451,11 +557,15 @@ impl AdminRpc for AdminRpcHandler { async fn list_zt_domains(self) -> Result<ListZtDomainsResponse> { let kv_store = self.state.kv_store(); let cert_resolver = &self.state.cert_resolver; + let certbot = &self.state.certbot; let domains = kv_store .list_zt_domain_configs() .into_iter() - .map(|config| zt_domain_to_proto(config, kv_store, cert_resolver)) + .map(|config| { + let records = certbot.required_dns_records(&config); + zt_domain_to_proto(config, kv_store, cert_resolver, records) + }) .collect(); Ok(ListZtDomainsResponse { domains }) @@ -465,58 +575,71 @@ impl AdminRpc for AdminRpcHandler { let kv_store = self.state.kv_store(); let cert_resolver = &self.state.cert_resolver; + let domain = normalize_zt_domain(&request.domain)?; let config = kv_store - .get_zt_domain_config(&request.domain) + .get_zt_domain_config(&domain) .context("ZT-Domain config not found")?; - Ok(zt_domain_to_proto(config, kv_store, cert_resolver)) + let records = self.state.certbot.required_dns_records(&config); + Ok(zt_domain_to_proto(config, kv_store, cert_resolver, records)) } async fn add_zt_domain(self, request: ProtoZtDomainConfig) -> Result<ZtDomainInfo> { let kv_store = self.state.kv_store(); let cert_resolver = &self.state.cert_resolver; - // Check if domain already exists - if kv_store.get_zt_domain_config(&request.domain).is_some() { - bail!("ZT-Domain config already exists: {}", request.domain); - } + // Nothing stored to preserve: an omitted challenge is the default. + let config = proto_to_zt_domain_config(&request, kv_store, None)?; - let config = proto_to_zt_domain_config(&request, kv_store)?; + // Uniqueness is checked after normalization so wildcard, case, and a + // trailing root dot cannot silently overwrite the same DNS name. + if kv_store.get_zt_domain_config(&config.domain).is_some() { + bail!("ZT-Domain config already exists: {}", config.domain); + } kv_store.save_zt_domain_config(&config)?; info!("Added ZT-Domain config: {}", config.domain); - Ok(zt_domain_to_proto(config, kv_store, cert_resolver)) + let records = self.state.certbot.required_dns_records(&config); + Ok(zt_domain_to_proto(config, kv_store, cert_resolver, records)) } async fn update_zt_domain(self, request: ProtoZtDomainConfig) -> Result<ZtDomainInfo> { let kv_store = self.state.kv_store(); let cert_resolver = &self.state.cert_resolver; - // Check if config exists - kv_store - .get_zt_domain_config(&request.domain) + // Read the stored record first: an omitted `challenge` means "leave it + // alone", and this is the only place its current value is known. Looked + // up on the normalized key rather than the caller's presentation. + let domain = normalize_zt_domain(&request.domain)?; + let stored = kv_store + .get_zt_domain_config(&domain) .context("ZT-Domain config not found")?; - let config = proto_to_zt_domain_config(&request, kv_store)?; + let config = proto_to_zt_domain_config(&request, kv_store, Some(stored.challenge))?; kv_store.save_zt_domain_config(&config)?; info!("Updated ZT-Domain config: {}", config.domain); - Ok(zt_domain_to_proto(config, kv_store, cert_resolver)) + let records = self.state.certbot.required_dns_records(&config); + Ok(zt_domain_to_proto(config, kv_store, cert_resolver, records)) } async fn delete_zt_domain(self, request: DeleteZtDomainRequest) -> Result<()> { let kv_store = self.state.kv_store(); - // Check if config exists - kv_store - .get_zt_domain_config(&request.domain) - .context("ZT-Domain config not found")?; + let domain = normalize_zt_domain(&request.domain)?; + // A corrupt config must still be deletable, so check for the record + // itself: get_zt_domain_config cannot tell missing from unreadable, + // and refusing would leave a corrupt record permanently stuck. + ensure!( + kv_store.zt_domain_config_exists(&domain), + "ZT-Domain config not found" + ); // Delete config (cert data, acme, attestations are kept for historical purposes) - kv_store.delete_zt_domain_config(&request.domain)?; - info!("Deleted ZT-Domain config: {}", request.domain); + kv_store.delete_zt_domain_config(&domain)?; + info!("Deleted ZT-Domain config: {domain}"); Ok(()) } @@ -565,6 +688,7 @@ impl AdminRpc for AdminRpcHandler { .map(|att| CertAttestationInfo { public_key: att.public_key, quote: att.quote, + attestation: att.attestation, generated_by: att.generated_by, generated_at: att.generated_at, }); @@ -575,6 +699,7 @@ impl AdminRpc for AdminRpcHandler { .map(|att| CertAttestationInfo { public_key: att.public_key, quote: att.quote, + attestation: att.attestation, generated_by: att.generated_by, generated_at: att.generated_at, }) @@ -591,40 +716,54 @@ impl AdminRpc for AdminRpcHandler { // ==================== Global Certbot Configuration ==================== async fn get_certbot_config(self) -> Result<CertbotConfigResponse> { - let config = self.state.kv_store().get_certbot_config(); + let config = self.state.kv_store().get_certbot_config()?; Ok(CertbotConfigResponse { renew_interval_secs: config.renew_interval.as_secs(), renew_before_expiration_secs: config.renew_before_expiration.as_secs(), renew_timeout_secs: config.renew_timeout.as_secs(), acme_url: config.acme_url, + issuer_domain_name: config.issuer_domain_name, }) } async fn set_certbot_config(self, request: SetCertbotConfigRequest) -> Result<()> { let kv_store = self.state.kv_store(); - let mut config = kv_store.get_certbot_config(); - - // Update only the fields that are specified - if let Some(secs) = request.renew_interval_secs { - config.renew_interval = Duration::from_secs(secs); - } - if let Some(secs) = request.renew_before_expiration_secs { - config.renew_before_expiration = Duration::from_secs(secs); - } - if let Some(secs) = request.renew_timeout_secs { - config.renew_timeout = Duration::from_secs(secs); - } - if let Some(url) = request.acme_url { - config.acme_url = url; - } - + let config = merge_certbot_config(kv_store.get_certbot_config(), request)?; kv_store.set_certbot_config(&config)?; info!( - "Updated certbot config: renew_interval={:?}, renew_before_expiration={:?}, renew_timeout={:?}, acme_url={:?}", + "Updated certbot config: renew_interval={:?}, renew_before_expiration={:?}, renew_timeout={:?}, acme_url={:?}, issuer_domain_name={:?}", config.renew_interval, config.renew_before_expiration, config.renew_timeout, - config.acme_url + config.acme_url, + config.issuer_domain_name + ); + Ok(()) + } + + // ==================== Tombstone GC Configuration ==================== + + async fn get_tombstone_gc_config(self) -> Result<TombstoneGcConfigResponse> { + Ok(match self.state.kv_store().get_tombstone_gc_config()? { + Some(stored) => TombstoneGcConfigResponse { + writes_per_collection: stored.writes_per_collection, + stored_in_kv: true, + }, + None => TombstoneGcConfigResponse { + writes_per_collection: self.state.config.sync.tombstone_gc_writes, + stored_in_kv: false, + }, + }) + } + + async fn set_tombstone_gc_config(self, request: SetTombstoneGcConfigRequest) -> Result<()> { + let config = GlobalTombstoneGcConfig { + writes_per_collection: request.writes_per_collection, + }; + self.state.kv_store().set_tombstone_gc_config(&config)?; + info!( + "updated tombstone GC config: writes_per_collection={}", + config.writes_per_collection ); Ok(()) } @@ -657,6 +796,17 @@ impl AdminRpc for AdminRpcHandler { .with_context(|| format!("instance {} not found", request.instance_id))?; Ok(port_policy_view_to_proto(view)) } + + async fn set_instance_ready(self, request: SetInstanceReadyRequest) -> Result<()> { + // Absent is not false. Over the JSON transport an omitted -- or + // misspelled, since unknown keys are ignored -- `ready` would decode to + // the field's default and quietly pull the instance out of rotation, + // answering 200 while doing the opposite of nothing. + let ready = request + .ready + .context("`ready` is required: say true to put the instance back into rotation, false to take it out")?; + self.state.lock().set_ready(&request.instance_id, ready) + } } fn port_policy_from_proto(proto: RpcPortPolicy) -> Result<PortPolicy> { @@ -700,6 +850,7 @@ fn port_policy_view_to_proto(view: PortPolicyView) -> GetInstancePortPolicyRespo fn build_store_status( name: &str, status: WaveKvNodeStatus, + links: &[wavekv::sync::PeerLinkStatus], get_peer_last_seen: &impl Fn(u32) -> Vec<(u32, u64)>, ) -> StoreSyncStatus { StoreSyncStatus { @@ -709,6 +860,9 @@ fn build_store_status( next_seq: status.next_seq, dirty: status.dirty, wal_enabled: status.wal, + digest: status.digest, + entries_merged: status.entries_merged, + entries_rejected: status.entries_rejected, peers: status .peers .into_iter() @@ -717,12 +871,15 @@ fn build_store_status( .into_iter() .map(|(node_id, timestamp)| LastSeenEntry { node_id, timestamp }) .collect(); + let link = links.iter().find(|l| l.id == p.id); ProtoPeerSyncStatus { id: p.id, local_ack: p.ack, - peer_ack: p.pack, - buffered_logs: p.logs as u64, + peer_ack: p.peer_ack, last_seen, + heard_from: p.heard_from, + digest_mismatches: link.map(|l| l.digest_mismatches).unwrap_or(0), + consecutive_failures: link.map(|l| l.consecutive_failures).unwrap_or(0), } }) .collect(), @@ -741,13 +898,6 @@ impl RpcCall<Proxy> for AdminRpcHandler { // ==================== Helper Functions ==================== -fn now_secs() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() -} - fn generate_cred_id() -> String { use std::time::SystemTime; let ts = SystemTime::now() @@ -789,10 +939,44 @@ fn redact_token(token: &str) -> String { } } +fn normalize_zt_domain(domain: &str) -> Result<String> { + let domain = domain.trim().trim_end_matches('.'); + let domain = domain + .strip_prefix("*.") + .unwrap_or(domain) + .to_ascii_lowercase(); + validate_zt_domain(&domain)?; + Ok(domain) +} + +fn validate_zt_domain(domain: &str) -> Result<()> { + if domain.is_empty() || domain.len() > 253 || !domain.is_ascii() { + bail!("domain must be a non-empty ASCII DNS name of at most 253 bytes"); + } + for label in domain.split('.') { + if label.is_empty() + || label.len() > 63 + || label.starts_with('-') + || label.ends_with('-') + || !label + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + { + bail!("domain contains an invalid DNS label"); + } + } + Ok(()) +} + /// Convert proto ZtDomainConfig to internal ZtDomainConfig +/// +/// `current` is the challenge already stored for this domain, on an update. +/// An omitted `challenge` resolves to it rather than to the default, so a caller +/// that predates the field leaves it alone instead of downgrading the domain. fn proto_to_zt_domain_config( proto: &ProtoZtDomainConfig, kv_store: &crate::kv::KvStore, + current: Option<ChallengeKind>, ) -> Result<ZtDomainConfig> { // Normalize dns_cred_id: treat empty string as None (use default) let dns_cred_id = proto @@ -804,16 +988,30 @@ fn proto_to_zt_domain_config( // Validate DNS credential if specified if let Some(ref cred_id) = dns_cred_id { kv_store - .get_dns_credential(cred_id) + .get_dns_credential(cred_id)? .context("specified dns credential not found")?; } - // Strip wildcard prefix if user entered it - let domain = proto - .domain - .strip_prefix("*.") - .unwrap_or(&proto.domain) - .to_string(); + let domain = normalize_zt_domain(&proto.domain)?; + if proto.port == 0 { + bail!("port must be between 1 and 65535"); + } + + let challenge = match proto.challenge.as_deref() { + // Absent means "leave it as it is". UpdateZtDomain replaces the whole + // record, so reading absence as the default would let any edit from a + // caller that does not know the field -- a cached dashboard bundle, a + // script, an older SDK -- downgrade a dns-persist-01 domain to dns-01 + // cluster-wide, after which its hand-published CAA naming + // dns-persist-01 refuses every order. On an add there is nothing to + // preserve, so it falls back to the historical default. + None => current.unwrap_or_default(), + // Present but empty is still the default: that is what an explicit + // proto3 zero value carries, and every ZT domain predates the choice. + Some("" | "dns-01") => ChallengeKind::Dns01, + Some("dns-persist-01") => ChallengeKind::DnsPersist01, + Some(other) => bail!("unsupported challenge {other:?}, expected dns-01 or dns-persist-01"), + }; Ok(ZtDomainConfig { domain, @@ -821,14 +1019,20 @@ fn proto_to_zt_domain_config( port: proto.port.try_into().context("port out of range")?, node: proto.node, priority: proto.priority, + challenge, }) } /// Convert internal ZtDomainConfig to proto ZtDomainInfo (with cert status) +/// +/// `required_dns_records` is best effort: rendering it needs the ACME account +/// URI, and a domain whose ACME client cannot be built yet still has to be +/// listable. It comes back empty in that case rather than failing the call. fn zt_domain_to_proto( config: ZtDomainConfig, kv_store: &crate::kv::KvStore, cert_resolver: &crate::cert_store::CertResolver, + required_dns_records: Vec<String>, ) -> ZtDomainInfo { // Get certificate data for status let cert_data = kv_store.get_cert_data(&config.domain); @@ -842,6 +1046,11 @@ fn zt_domain_to_proto( loaded_in_memory, }); + let challenge = match config.challenge { + ChallengeKind::Dns01 => "dns-01", + ChallengeKind::DnsPersist01 => "dns-persist-01", + }; + ZtDomainInfo { config: Some(ProtoZtDomainConfig { domain: config.domain, @@ -849,7 +1058,421 @@ fn zt_domain_to_proto( port: config.port.into(), node: config.node, priority: config.priority, + challenge: Some(challenge.to_string()), }), cert_status, + required_dns_records, + } +} + +/// Apply a partial certbot-config update to the stored record. +/// +/// SetCertbotConfig is a merge: a field the operator leaves unset keeps its +/// stored value. That needs a readable base, and `global/certbot_config` is a +/// singleton with no delete RPC — so if an unreadable record simply failed the +/// call, the corruption would be permanent, and since `do_rotate_acme_credentials` +/// reads the same key it would keep RotateAcmeCredentials blocked along with it. +/// +/// Merging into the defaults instead is not the answer either: `acme_url` +/// defaults to empty, which means Let's Encrypt production. An operator who hit +/// a corrupt record and then tuned `renew_interval` would silently move issuance +/// off their staging or private ACME server and start burning real rate limits — +/// exactly the switch the fail-closed reader exists to prevent. +/// +/// So an unreadable record is repairable, but only by a request that states +/// every field. Nothing is ever inherited from a record we cannot read. +fn merge_certbot_config( + stored: Result<GlobalCertbotConfig>, + request: SetCertbotConfigRequest, +) -> Result<GlobalCertbotConfig> { + let mut config = match stored { + Ok(config) => config, + Err(err) => { + ensure!( + request.renew_interval_secs.is_some() + && request.renew_before_expiration_secs.is_some() + && request.renew_timeout_secs.is_some() + && request.acme_url.is_some() + && request.issuer_domain_name.is_some(), + "the stored certbot config is unreadable ({err:#}), so it can only be \ + replaced as a whole: resend with renew_interval_secs, \ + renew_before_expiration_secs, renew_timeout_secs, acme_url and \ + issuer_domain_name all set" + ); + warn!("certbot config is unreadable ({err:#}); replacing it wholesale"); + GlobalCertbotConfig::default() + } + }; + + // Update only the fields that are specified + if let Some(secs) = request.renew_interval_secs { + config.renew_interval = Duration::from_secs(secs); + } + if let Some(secs) = request.renew_before_expiration_secs { + config.renew_before_expiration = Duration::from_secs(secs); + } + if let Some(secs) = request.renew_timeout_secs { + // Zero would abort every order before it starts, and the DNS wait is + // derived from this value. + ensure!(secs > 0, "renew_timeout_secs must be greater than zero"); + config.renew_timeout = Duration::from_secs(secs); + } + if let Some(url) = request.acme_url { + config.acme_url = url; + } + if let Some(name) = request.issuer_domain_name { + // Checked here rather than where it is used: it is written verbatim into + // CAA records, and those are rewritten by deleting the old ones first, + // so a value that cannot be a DNS name takes the zone's issuance + // permission down with it. Stored as configured -- empty keeps meaning + // the default -- but only once it is known to be usable. + certbot::resolve_issuer_domain_name(&name).context("invalid issuer_domain_name")?; + config.issuer_domain_name = name; + } + Ok(config) +} + +#[cfg(test)] +mod certbot_config_tests { + use super::*; + + fn stored() -> GlobalCertbotConfig { + GlobalCertbotConfig { + renew_interval: Duration::from_secs(3600), + acme_url: "https://acme-staging.example/directory".to_string(), + ..Default::default() + } + } + + #[test] + fn a_partial_update_keeps_the_fields_it_does_not_mention() { + let merged = merge_certbot_config( + Ok(stored()), + SetCertbotConfigRequest { + renew_timeout_secs: Some(60), + ..Default::default() + }, + ) + .expect("a readable record merges"); + assert_eq!(merged.renew_timeout, Duration::from_secs(60)); + assert_eq!(merged.acme_url, stored().acme_url); + } + + #[test] + fn a_partial_update_cannot_repair_an_unreadable_record() { + // Falling back to the defaults here would reset `acme_url` to empty, + // silently moving issuance to Let's Encrypt production. + let err = merge_certbot_config( + Err(anyhow::anyhow!("corrupt record")), + SetCertbotConfigRequest { + renew_interval_secs: Some(60), + ..Default::default() + }, + ) + .expect_err("a partial update must not inherit from an unreadable record"); + assert!(err.to_string().contains("acme_url"), "{err:#}"); + } + + #[test] + fn a_complete_request_replaces_an_unreadable_record() { + // The only repair path: no field is inherited, so nothing is guessed. + let merged = merge_certbot_config( + Err(anyhow::anyhow!("corrupt record")), + SetCertbotConfigRequest { + renew_interval_secs: Some(60), + renew_before_expiration_secs: Some(86400), + renew_timeout_secs: Some(30), + acme_url: Some("https://acme-staging.example/directory".to_string()), + issuer_domain_name: Some("pebble.letsencrypt.org".to_string()), + }, + ) + .expect("a complete request replaces the record"); + assert_eq!(merged.renew_interval, Duration::from_secs(60)); + assert_eq!(merged.acme_url, "https://acme-staging.example/directory"); + assert_eq!(merged.issuer_domain_name, "pebble.letsencrypt.org"); + } + + /// Repairing from the defaults would reset the issuer name to empty -- + /// Let's Encrypt -- for a deployment pointed at another CA, which then + /// publishes CAA and validation records naming a CA its orders never reach. + /// That is the same hazard `acme_url` is in the required set for. + #[test] + fn repairing_an_unreadable_record_cannot_forget_the_issuer_name() { + let err = merge_certbot_config( + Err(anyhow::anyhow!("corrupt record")), + SetCertbotConfigRequest { + renew_interval_secs: Some(60), + renew_before_expiration_secs: Some(86400), + renew_timeout_secs: Some(30), + acme_url: Some("https://pebble.example/dir".to_string()), + issuer_domain_name: None, + }, + ) + .expect_err("a repair that omits the issuer name must be refused"); + assert!(err.to_string().contains("issuer_domain_name"), "{err:#}"); + } + + /// The name is written verbatim into CAA records, and writing them deletes + /// the records they replace, so a value that cannot be a DNS name is + /// refused where it is set rather than where it would corrupt a zone. + #[test] + fn a_malformed_issuer_name_is_refused() { + let err = merge_certbot_config( + Ok(stored()), + SetCertbotConfigRequest { + issuer_domain_name: Some("lets encrypt.org".to_string()), + ..Default::default() + }, + ) + .expect_err("a name with a space must be refused"); + assert!(err.to_string().contains("issuer_domain_name"), "{err:#}"); + } + + /// Zero would abort every order before it began, and the DNS wait is + /// derived from this value. + #[test] + fn a_zero_renew_timeout_is_refused() { + let err = merge_certbot_config( + Ok(stored()), + SetCertbotConfigRequest { + renew_timeout_secs: Some(0), + ..Default::default() + }, + ) + .expect_err("a zero renew timeout must be refused"); + assert!(err.to_string().contains("renew_timeout_secs"), "{err:#}"); + } + + /// The name the CA is known by has to be settable, or a deployment pointed + /// at a non-Let's-Encrypt ACME server publishes records naming the wrong CA + /// with no way to correct them. + #[test] + fn the_issuer_domain_name_round_trips() { + let merged = merge_certbot_config( + Ok(stored()), + SetCertbotConfigRequest { + issuer_domain_name: Some("pebble.letsencrypt.org".to_string()), + ..Default::default() + }, + ) + .expect("a partial update keeps the rest"); + assert_eq!(merged.issuer_domain_name, "pebble.letsencrypt.org"); + assert_eq!(merged.acme_url, stored().acme_url); + } +} + +/// An omitted `challenge` has to mean "leave it as it is", or every caller that +/// predates the field silently downgrades a dns-persist-01 domain. +#[cfg(test)] +mod zt_domain_challenge_tests { + use super::*; + + fn kv_store(dir: &std::path::Path) -> crate::kv::KvStore { + crate::kv::KvStore::new(1, vec![], dir, None).expect("failed to create kv store") + } + + fn request(challenge: Option<&str>) -> ProtoZtDomainConfig { + ProtoZtDomainConfig { + domain: "example.com".to_string(), + dns_cred_id: None, + port: 443, + node: None, + priority: 7, + challenge: challenge.map(ToString::to_string), + } + } + + /// The case the field is `optional` for: a cached dashboard bundle, a curl + /// script, or an older SDK edits an unrelated field and must not take the + /// domain's challenge down with it. + #[test] + fn an_omitted_challenge_keeps_the_stored_one() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let config = proto_to_zt_domain_config( + &request(None), + &kv_store(dir.path()), + Some(ChallengeKind::DnsPersist01), + ) + .expect("an update without a challenge is valid"); + assert_eq!(config.challenge, ChallengeKind::DnsPersist01); + assert_eq!(config.priority, 7); + } + + /// On an add there is no stored value to preserve, so absence is the + /// historical default rather than an error. + #[test] + fn an_omitted_challenge_on_an_add_is_dns01() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let config = proto_to_zt_domain_config(&request(None), &kv_store(dir.path()), None) + .expect("an add without a challenge is valid"); + assert_eq!(config.challenge, ChallengeKind::Dns01); + } + + /// Preserving an omitted value must not make the field unsettable: a caller + /// that names dns-01 is asking to switch back, and gets it. + #[test] + fn an_explicit_challenge_overrides_the_stored_one() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let store = kv_store(dir.path()); + let config = proto_to_zt_domain_config( + &request(Some("dns-01")), + &store, + Some(ChallengeKind::DnsPersist01), + ) + .expect("an explicit challenge is valid"); + assert_eq!(config.challenge, ChallengeKind::Dns01); + + let config = proto_to_zt_domain_config( + &request(Some("dns-persist-01")), + &store, + Some(ChallengeKind::Dns01), + ) + .expect("an explicit challenge is valid"); + assert_eq!(config.challenge, ChallengeKind::DnsPersist01); + } + + /// An explicit empty string is what a proto3 zero value carries, and every + /// ZT domain predates the choice, so it still reads as the default. + #[test] + fn an_explicitly_empty_challenge_is_the_default() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let config = proto_to_zt_domain_config( + &request(Some("")), + &kv_store(dir.path()), + Some(ChallengeKind::DnsPersist01), + ) + .expect("an empty challenge is valid"); + assert_eq!(config.challenge, ChallengeKind::Dns01); + } + + /// An unrecognized value is refused rather than silently defaulted: a typo + /// that read as dns-01 would be the downgrade this field exists to prevent. + #[test] + fn an_unknown_challenge_is_refused() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let err = proto_to_zt_domain_config( + &request(Some("dns-persist-02")), + &kv_store(dir.path()), + None, + ) + .expect_err("an unknown challenge must be refused"); + assert!(err.to_string().contains("dns-persist-02"), "{err:#}"); + } +} + +#[cfg(test)] +mod zt_domain_tests { + use super::validate_zt_domain; + + #[test] + fn accepts_a_dns_domain() { + validate_zt_domain("service.example.com").unwrap(); + } + + #[test] + fn rejects_empty_and_invalid_dns_domains() { + for domain in [ + "", + ".example.com", + "example..com", + "-bad.example", + "bad-.example", + ] { + assert!( + validate_zt_domain(domain).is_err(), + "{domain} should be rejected" + ); + } + } +} + +#[cfg(test)] +mod wavekv_status_tests { + use super::{build_store_status, WaveKvNodeStatus}; + use wavekv::{node::PeerStatus, sync::PeerLinkStatus}; + + #[test] + fn wavekv_status_preserves_store_and_peer_telemetry() { + let status = WaveKvNodeStatus { + id: 1, + n_kvs: 3, + next_seq: 11, + dirty: true, + wal: true, + digest: "deadbeef".to_string(), + entries_merged: 17, + entries_rejected: 2, + peers: vec![PeerStatus { + id: 7, + ack: 5, + peer_ack: 4, + heard_from: true, + }], + }; + let links = vec![PeerLinkStatus { + id: 7, + protocol: "v2", + digest_mismatches: 3, + consecutive_failures: 6, + }]; + + let proto = build_store_status("persistent", status, &links, &|peer| { + assert_eq!(peer, 7); + vec![(2, 1234)] + }); + + assert_eq!(proto.name, "persistent"); + assert_eq!(proto.node_id, 1); + assert_eq!(proto.n_keys, 3); + assert_eq!(proto.next_seq, 11); + assert!(proto.dirty); + assert!(proto.wal_enabled); + assert_eq!(proto.digest, "deadbeef"); + assert_eq!(proto.entries_merged, 17); + assert_eq!(proto.entries_rejected, 2); + assert_eq!(proto.peers.len(), 1); + + let peer = &proto.peers[0]; + assert_eq!(peer.id, 7); + assert_eq!(peer.local_ack, 5); + assert_eq!(peer.peer_ack, 4); + assert!(peer.heard_from); + assert_eq!(peer.digest_mismatches, 3); + assert_eq!(peer.consecutive_failures, 6); + assert_eq!(peer.last_seen.len(), 1); + assert_eq!(peer.last_seen[0].node_id, 2); + assert_eq!(peer.last_seen[0].timestamp, 1234); + } +} + +#[cfg(test)] +mod set_instance_ready_tests { + use dstack_gateway_rpc::SetInstanceReadyRequest; + + /// prpc gives every generated field `#[serde(default)]`, and serde ignores + /// keys it does not know. A plain proto3 `bool ready` therefore turns both + /// "I forgot the field" and "I misspelled the field" into a 200 that pulls + /// the instance out of rotation. `optional` keeps absent distinguishable so + /// the handler can refuse it. + #[test] + fn an_omitted_ready_stays_distinguishable_from_a_gate_off() { + let omitted: SetInstanceReadyRequest = + serde_json::from_str(r#"{"instance_id": "abc"}"#).expect("decodes"); + assert_eq!(omitted.ready, None, "absent must not decode as false"); + + let misspelled: SetInstanceReadyRequest = + serde_json::from_str(r#"{"instance_id": "abc", "redy": true}"#).expect("decodes"); + assert_eq!( + misspelled.ready, None, + "an unknown key is dropped, so this is the omitted case too" + ); + + let explicit: SetInstanceReadyRequest = + serde_json::from_str(r#"{"instance_id": "abc", "ready": false}"#).expect("decodes"); + assert_eq!( + explicit.ready, + Some(false), + "a stated false still gates off" + ); } } diff --git a/dstack/gateway/src/cert_store.rs b/dstack/gateway/src/cert_store.rs index 16a4f76d0..d5ee987b6 100644 --- a/dstack/gateway/src/cert_store.rs +++ b/dstack/gateway/src/cert_store.rs @@ -75,6 +75,7 @@ impl CertStore { } /// Get certificate data for a domain + #[cfg(test)] pub fn get_cert_data(&self, domain: &str) -> Option<&CertData> { self.cert_data.get(domain) } @@ -172,19 +173,9 @@ impl CertResolver { let old_store = self.get(); - // Build new store with all existing certs plus the new/updated one - let mut builder = CertStoreBuilder::new(); - - // Copy existing certs (except the one we're replacing) - for existing_domain in old_store.list_domains() { - if existing_domain != domain { - if let Some(existing_data) = old_store.get_cert_data(&existing_domain) { - builder.add_cert(&existing_domain, existing_data)?; - } - } - } - - // Add the new/updated cert + // Clone the installed store without revalidating existing certificates. An expired + // certificate for one domain must not block another domain from being renewed. + let mut builder = CertStoreBuilder::from_store(&old_store); builder.add_cert(domain, data)?; // Atomically swap @@ -233,11 +224,25 @@ impl CertStoreBuilder { } } + fn from_store(store: &CertStore) -> Self { + Self { + exact_certs: store.exact_certs.clone(), + wildcard_certs: store.wildcard_certs.clone(), + cert_data: store.cert_data.clone(), + } + } + /// Add a certificate to the builder /// /// The domain is the base domain (e.g., "example.com"). /// All gateway certificates are wildcard certs for "*.{domain}". pub fn add_cert(&mut self, domain: &str, data: &CertData) -> Result<()> { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .context("system time is before Unix epoch")? + .as_secs(); + anyhow::ensure!(data.not_after > now, "certificate is expired"); + let certified_key = parse_certified_key(&data.cert_pem, &data.key_pem) .with_context(|| format!("failed to parse certificate for {}", domain))?; @@ -291,7 +296,11 @@ fn parse_certified_key(cert_pem: &str, key_pem: &str) -> Result<CertifiedKey> { let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key) .map_err(|e| anyhow::anyhow!("failed to create signing key: {:?}", e))?; - Ok(CertifiedKey::new(certs, signing_key)) + let certified_key = CertifiedKey::new(certs, signing_key); + certified_key + .keys_match() + .context("certificate and private key do not match")?; + Ok(certified_key) } /// Format expiry timestamp as human-readable string @@ -435,4 +444,82 @@ mod tests { // Should not resolve different domain assert!(!store.has_cert_for_sni("example.org")); } + + #[test] + fn mismatched_key_update_retains_previous_certificate() { + let original = make_test_cert_data(); + let mut mismatched = make_test_cert_data(); + mismatched.cert_pem = original.cert_pem.clone(); + + let resolver = CertResolver::new(); + resolver + .update_cert("example.com", &original) + .expect("failed to install original certificate"); + let original_not_after = resolver + .get() + .get_cert_data("example.com") + .expect("original certificate missing") + .not_after; + + let error = resolver + .update_cert("example.com", &mismatched) + .expect_err("mismatched key must be rejected"); + assert!(error.to_string().contains("failed to parse certificate")); + assert_eq!( + resolver + .get() + .get_cert_data("example.com") + .expect("original certificate was lost") + .not_after, + original_not_after + ); + assert!(resolver.get().has_cert_for_sni("app.example.com")); + } + + #[test] + fn expired_certificate_does_not_block_another_domain_update() { + let mut builder = CertStoreBuilder::new(); + builder + .add_cert("expired.example.com", &make_test_cert_data()) + .expect("failed to install initial certificate"); + builder + .cert_data + .get_mut("expired.example.com") + .expect("installed certificate is missing") + .not_after = 1; + + let resolver = CertResolver::new(); + resolver.set(Arc::new(builder.build())); + resolver + .update_cert("fresh.example.com", &make_test_cert_data()) + .expect("expired certificate blocked an unrelated update"); + + assert!(resolver.get().has_cert("expired.example.com")); + assert!(resolver.get().has_cert("fresh.example.com")); + } + + #[test] + fn expired_update_retains_previous_certificate() { + let original = make_test_cert_data(); + let mut expired = make_test_cert_data(); + expired.not_after = 1; + + let resolver = CertResolver::new(); + resolver + .update_cert("example.com", &original) + .expect("failed to install original certificate"); + resolver + .update_cert("example.com", &expired) + .expect_err("expired certificate must be rejected"); + + assert_eq!( + resolver + .get() + .get_cert_data("example.com") + .expect("original certificate was lost") + .not_after, + original.not_after + ); + assert!(resolver.get().has_cert_for_sni("app.example.com")); + } } diff --git a/dstack/gateway/src/config.rs b/dstack/gateway/src/config.rs index e43bf54bd..7dec16a7b 100644 --- a/dstack/gateway/src/config.rs +++ b/dstack/gateway/src/config.rs @@ -4,13 +4,14 @@ use anyhow::{bail, Context, Result}; use cmd_lib::run_cmd as cmd; +use dstack_attest::attestation::AttestationVerifierConfig; use ipnet::Ipv4Net; use load_config::load_config; use rocket::figment::Figment; use serde::{Deserialize, Serialize}; -use std::net::Ipv4Addr; +use std::net::{Ipv4Addr, SocketAddr}; use std::path::PathBuf; -use std::time::Duration; +use std::time::{Duration, Instant}; use tracing::info; #[derive(Debug, Clone, Deserialize)] @@ -30,6 +31,48 @@ impl WgConfig { fn validate(&self) -> Result<()> { validate(self.ip, &self.reserved_net, self.client_ip_range) } + + /// Whether this gateway may allocate `ip` to a CVM registering with it. + /// + /// Narrower than [`Self::is_routable_client_ip`]: `client_ip_range` is this + /// node's *share* of the cluster's address space, and handing out an address + /// from outside it would collide with whichever node owns that share. + pub fn is_valid_client_ip(&self, ip: Ipv4Addr) -> bool { + self.client_ip_range.contains(&ip) && self.is_routable_client_ip(ip) + } + + /// Whether `ip` may appear as a WireGuard peer address on this gateway. + /// + /// Deliberately says nothing about *which pool* the address came from. A + /// CVM registers with one gateway but is handed every gateway as a + /// WireGuard server, so each node carries peers for the CVMs registered on + /// the other nodes — and each node allocates from its own + /// `client_ip_range`. Nothing in this node's config describes the other + /// nodes' pools, and the deployments do not even agree on a shape that + /// could be inferred: `dstack-app/deploy-to-vmm.sh` puts every pool inside + /// one /16 that each interface covers, while the `test-run` suites give + /// each node a /24 that no other node's interface covers. + /// Judging a replicated address by local topology refuses legitimate peers + /// under the second shape, so this is limited to what a node can assert on + /// its own: an ordinary unicast address that is not one of *this* gateway's. + /// + /// What keeps the peer list coherent is not this check but the uniqueness + /// pass in `kv::import` — no two instances may claim the same address — + /// which holds cluster-wide because it runs over the whole KV contents. + pub fn is_routable_client_ip(&self, ip: Ipv4Addr) -> bool { + if ip.is_unspecified() || ip.is_loopback() || ip.is_multicast() || ip.is_broadcast() { + return false; + } + // This gateway's own addresses: handing them to a peer would point the + // interface's traffic into a tunnel. + if self.ip.addr() == ip || self.ip.broadcast() == ip { + return false; + } + if self.reserved_net.iter().any(|net| net.contains(&ip)) { + return false; + } + true + } } fn validate(ip: Ipv4Net, reserved_net: &[Ipv4Net], client_ip_range: Ipv4Net) -> Result<()> { @@ -102,6 +145,23 @@ where } } +fn default_true() -> bool { + true +} + +/// Roughly hourly on a busy ephemeral store and rarer on the quiet persistent +/// one -- the cadence follows write volume, which is also what produces +/// tombstones. Any value works for correctness: the ack watermark decides what +/// may go, not this, so it trades disk against how often every node stops to +/// scan its data map. +fn default_tombstone_gc_writes() -> u64 { + 10_000 +} + +fn default_handshake_stale() -> Duration { + Duration::from_secs(30 * 60) +} + #[derive(Debug, Clone, Deserialize)] pub struct ProxyConfig { pub tls_crypto_provider: CryptoProvider, @@ -110,10 +170,61 @@ pub struct ProxyConfig { #[serde(deserialize_with = "deserialize_port_range")] pub listen_port: Vec<u16>, pub timeouts: Timeouts, + /// Relay buffer size, per direction, for connections that copy through + /// userspace -- TLS terminate, and passthrough before the splice gate. + /// + /// Costs `2 * buffer_size` of address space per such connection, of which + /// only the pages actually touched become resident: measured at 2 000 + /// concurrent streaming connections, the userspace relay path sat at ~52 KB + /// RSS per connection. Budget for it before raising this on a gateway that + /// fronts many idle-ish connections; the same measurement with kTLS, where + /// the payload is spliced and never enters the process, was ~12 KB. + /// + /// 64 KiB is the bulk-throughput sweet spot: it is large enough to keep a + /// 1 MiB pipe fed without the syscall rate 8 KiB imposed. pub buffer_size: usize, pub connect_top_n: usize, - pub localhost_enabled: bool, pub workers: usize, + /// Run one single-threaded runtime per worker, each with its own + /// `SO_REUSEPORT` listener, instead of one accept thread feeding a shared + /// work-stealing runtime. + /// + /// A connection is then accepted and served entirely on one thread. The + /// default model costs ~0.6 context switches per request (accept-thread + /// handoff plus work-stealing migrations); HAProxy's thread-per-core design + /// measures ~0. Linux-only (needs SO_REUSEPORT). + #[serde(default)] + pub thread_per_core: bool, + /// Hand a freshly accepted connection to a less loaded core when the + /// accepting one is running ahead. + /// + /// `SO_REUSEPORT` picks a listener by hashing the connection's 4-tuple and + /// thread-per-core cannot move a connection afterwards, so a core can sit + /// starved: measured at 16 connections over 4 cores, utilisation came out + /// `[99, 42, 101, 101]`. Rebalancing costs one channel send per *rebalanced + /// connection*, never per request. Requires `thread_per_core`. + /// + /// On by default. It used to be opt-in, because handing a connection over + /// cost 2-3% wherever the hash was already even -- that turned out to be a + /// bug (the migrated socket kept its registration on the accepting core's + /// reactor), and with it fixed the trade is one-sided. Measured on a 4-core + /// gateway, 3-5 runs per arm after warmup: + /// + /// | workload | off | on | | + /// |---|---|---|---| + /// | passthrough small-request, 8 conns | 113 350 | 143 341 | **+26.5%** | + /// | passthrough small-request, 16 conns | 231 043 | 258 787 | **+12.0%** | + /// | passthrough small-request, 50 conns | 294 937 | 295 311 | +0.1% | + /// | TLS terminate small-request, 50 conns | 244 304 | 246 228 | +0.8% | + /// | TLS terminate, connections/s | 41 024 | 43 019 | +4.9% | + /// | passthrough, connections/s | 17 431 | 17 319 | -0.6% | + /// | passthrough, bulk throughput | 12.10 GB/s | 12.16 GB/s | +0.5% | + /// + /// The gain is largest where the hash has fewest connections to spread and + /// the worst-loaded core would otherwise starve: at 16 connections the + /// quietest core goes from 63% to 97% busy. + #[serde(default = "default_true")] + pub connection_rebalance: bool, #[serde(default)] pub base_domain: Option<String>, #[serde(default)] @@ -122,6 +233,10 @@ pub struct ProxyConfig { pub cert_key: Option<PathBuf>, pub app_address_ns_prefix: String, pub app_address_ns_compat: bool, + /// Dedicated DNS servers for app-address TXT lookups. + /// The system resolver is used when this list is empty. + #[serde(default)] + pub app_address_dns_servers: Vec<SocketAddr>, /// Maximum concurrent connections per app. 0 means unlimited. pub max_connections_per_app: u64, /// Port the dstack guest-agent listens on inside each CVM. Used by the @@ -131,8 +246,228 @@ pub struct ProxyConfig { /// (e.g. when behind a PP-aware load balancer like Cloudflare). #[serde(default)] pub inbound_pp_enabled: bool, + /// Use `splice(2)` zero-copy relaying for the TLS-passthrough path. Both + /// sides are raw TCP there, so payload never needs to enter userspace. + /// Linux-only; ignored for the TLS-terminate path. + /// + /// Absent disables splice entirely; see [`EngageAfter`] for what a present + /// section means. + /// + /// Tradeoff (measured on a 4-core gateway): bulk passthrough throughput + /// +~12% with a lower tail latency under load, but small-request latency + /// regresses (each tiny message pays an extra pipe hop). splice costs ~17 + /// syscalls per connection to move a small response (fill pipe, drain pipe, + /// readiness retries) where a read/write pair needs two: its benefit is per + /// byte, its cost is per connection, which is what the gates amortise. + #[serde(default)] + pub tcp_splice: Option<SpliceConfig>, + /// Offload TLS record encryption to the kernel (kTLS) on the + /// TLS-terminate path. The handshake still runs in rustls; only the + /// symmetric crypto moves into the kernel afterwards. Linux-only. + /// + /// Absent disables kTLS entirely; see [`EngageAfter`] for what a present + /// section means. Gated offload additionally requires `tcp_splice`, since + /// the point of handing the socket to the kernel is to then splice it. + /// + /// A kernel built without `CONFIG_TLS` cannot honour this, so startup + /// probes for the TLS ULP and clears this section with a warning if it is + /// missing, rather than letting every connection discover it at the gate. + /// + /// kTLS costs ~30% of connection setup rate but wins ~25% on bulk + /// throughput, so paying the setup cost up front is wrong for short + /// request/response connections. + /// + /// On token-streaming traffic the throughput win does not materialise, but + /// a memory win does. Measured on the terminate path, 10k connections with + /// 2k streaming a 64 B record every 25 ms, 2 runs per arm: + /// + /// | | userspace rustls | kTLS | + /// |---|---|---| + /// | latency p50 | 0.174 / 0.181 ms | 0.180 / 0.174 ms | + /// | latency p99 | 0.602 / 0.526 ms | 0.859 / 0.537 ms | + /// | **RSS** | **349 MB** | **206 MB** | + /// + /// Latency is unchanged, as expected: at 64 B per record the per-record + /// overhead dominates and there is almost no symmetric crypto to move into + /// the kernel. The 41% RSS drop is the real effect and was not predicted -- + /// with kTLS the payload is spliced without ever entering this process, so + /// the per-connection userspace relay buffers disappear (~14 KB/connection + /// here). Weigh that against handing session keys to the kernel. + /// + /// Security note: enabling this hands the negotiated session keys to the + /// kernel via `dangerous_extract_secrets`, so the keys live outside + /// rustls' control. Inside a CVM the kernel is part of the measured TCB, + /// but on a non-TEE host this widens key exposure. Off by default. + #[serde(default)] + pub ktls: Option<EngageAfter>, /// Background lazy-fetch behaviour for `port_policy` (legacy CVMs). pub port_policy_fetch: PortPolicyFetchConfig, + pub health_check: HealthCheckConfig, +} + +impl ProxyConfig { + /// The idle window every relay enforces, or `None` when data timeouts are + /// off. Computed in one place so the buffered bridge and the gated fast + /// paths cannot end up enforcing different things. + pub fn idle_timeout(&self) -> Option<Duration> { + self.timeouts + .data_timeout_enabled + .then_some(self.timeouts.idle) + } +} + +/// Configuration for `splice(2)` relaying on the TLS-passthrough path. +#[derive(Debug, Clone, Default, Deserialize)] +pub struct SpliceConfig { + /// When splice should take over from the buffered relay. + #[serde(flatten)] + pub engage: EngageAfter, + /// Return a splice pipe to the thread-local pool while waiting for the + /// next chunk, instead of holding it for the connection's lifetime. + /// + /// A relay holds one pipe per direction, so a spliced connection pins four + /// descriptors. Held for the connection's lifetime that is proportional to + /// *connections*: 50k streaming connections need 300k descriptors. Released + /// while idle it is proportional to *chunks actually in flight*, which for + /// bursty traffic is far smaller -- LLM token streaming moves a ~64 B record + /// every 25 ms and spends over 99% of the connection idle. + /// + /// The pipe is provably empty at the point it is released: the relay only + /// waits for readability after the previous chunk has been fully drained + /// into the destination, so nothing is left to corrupt the next borrower. + /// + /// The cost is a pool pop and push per idle-to-active transition, both + /// `Vec` operations on a thread-local. Pipes are only created when the pool + /// is empty, so the live pipe count converges on the peak number of + /// concurrent in-flight chunks rather than churning `pipe2`/`close`. + /// + /// Measured on a 4-core gateway streaming a 64 B record every 25 ms per + /// connection, 2 runs per arm at each scale: + /// + /// | | off | on | + /// |---|---|---| + /// | pipe fds, 2k connections | 8 000 | **8** | + /// | pipe fds, 50k conns / 10k streams | 53 864 / 55 212 | **8** | + /// | RSS, 50k connections | 1 203 MB | 1 202 / 1 204 MB | + /// | latency p50, 50k | 20.9 / 20.3 ms | 21.6 / 21.7 ms | + /// | latency p999, 50k | 111 / 113 ms | 68 / 79 ms | + /// + /// The descriptor result is the point, and it is flat in the connection + /// count: 8 descriptors is four pipes, one per worker thread, for the whole + /// gateway, at 2k connections and at 50k alike. That is the bound this knob + /// exists to impose. + /// + /// Latency is close to a wash and should not be used to justify the knob. At + /// 2k connections there was no consistent difference at all (the single + /// fastest run of seven was an `off` run). At 50k, where the box is + /// saturated, `on` costs ~1 ms on p50 and saves ~40 ms on p999; the p999 + /// gain is consistent across repeats but comes from a regime that is already + /// over budget. RSS is unchanged either way. + /// + /// Mixing bulk transfers with token streams does not break the pooling, and + /// this was the failure worth checking: a connection that is never idle + /// never reaches the release point, so bulk traffic could in principle hold + /// every pipe and force each token to allocate a fresh one. Measured with + /// 10k streaming connections alongside 256 bulk connections saturating the + /// passthrough path at 12.8 GB/s, 2 runs per arm: + /// + /// | | off | on | + /// |---|---|---| + /// | pipe fds | 9 024 (= 2000*4 + 256*4) | **8** | + /// | bulk throughput | 12.8 GB/s | 12.8 GB/s | + /// | stream latency p50 | 0.099 / 0.102 ms | 0.093 / 0.089 ms | + /// | bulk latency p999 | 14.3 / 11.4 ms | 6.9 / 7.7 ms | + /// + /// The pool never degenerates because even a saturated bulk connection is + /// idle for tens of microseconds between chunks (measured inter-record gap + /// 62 us) and releases in that window. Throughput is identical and both + /// traffic classes are slightly better off with `on`, so there is no bulk + /// regression to trade against the descriptor saving. + #[serde(default)] + pub release_idle_pipes: bool, +} + +/// When an adaptive optimisation should engage on a connection. +/// +/// Both gates are optional and independent, and the optimisation engages as +/// soon as *either* fires. They catch different traffic and neither subsumes +/// the other: +/// +/// - `after_bytes` catches high-rate connections almost immediately -- a bulk +/// transfer trips a 64 KiB gate within milliseconds -- but is blind to +/// long-lived low-rate streams. LLM token streaming at 40 tok/s of ~64 B +/// records needs ~25 s of wall time to move 64 KiB, so a byte gate leaves the +/// whole early phase of every stream on the copy path, and never promotes +/// short conversations at all. +/// Measured: with a 64 KiB byte gate alone, connections streaming a 64 B +/// record every 25 ms promoted 25 s after they started streaming, matching +/// `65536 / (64 B * 40/s)`. Adding `after_duration = "5s"` moved that to 5 s, +/// a 5x earlier handover, with no change in added latency or RSS. +/// - `after_duration` catches exactly those long-lived low-rate streams, but is +/// blind to short high-rate ones, which finish before it fires. +/// +/// With neither gate set there is nothing to wait for, so the optimisation +/// engages from the first byte. "Never engage" is expressed by omitting the +/// whole section rather than by a sentinel value here, so every state has +/// exactly one representation. +#[derive(Debug, Clone, Default, Deserialize)] +pub struct EngageAfter { + /// Bytes the connection must transfer first. Absent = this gate never + /// fires. + #[serde(default)] + pub after_bytes: Option<u64>, + /// Wall time the connection must stay alive first, measured from the point + /// the relay starts (upstream already connected). Absent = this gate never + /// fires. + #[serde(default, with = "serde_duration::option")] + pub after_duration: Option<Duration>, +} + +impl EngageAfter { + /// No gate configured, so there is nothing to wait for. + pub fn is_immediate(&self) -> bool { + self.after_bytes.is_none() && self.after_duration.is_none() + } + + /// Whether either gate has been reached. + /// + /// `start` is only read when the duration gate is configured, so a + /// bytes-only config pays no clock read per message. + pub fn reached(&self, moved: u64, start: Instant) -> bool { + self.after_bytes.is_some_and(|bytes| moved >= bytes) + || self + .after_duration + .is_some_and(|limit| start.elapsed() >= limit) + } +} + +/// Rendered for the dashboard and the `Status` RPC, so it reads as the answer to +/// "when does this engage?" rather than as a struct dump. +impl std::fmt::Display for EngageAfter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match (self.after_bytes, self.after_duration) { + (None, None) => write!(f, "immediate"), + (Some(bytes), None) => write!(f, "after {}", DisplayBytes(bytes)), + (None, Some(after)) => write!(f, "after {after:?}"), + (Some(bytes), Some(after)) => write!(f, "after {} or {after:?}", DisplayBytes(bytes)), + } + } +} + +/// A byte threshold in the units the config file writes it in: binary units +/// when they divide evenly, raw bytes otherwise. +struct DisplayBytes(u64); + +impl std::fmt::Display for DisplayBytes { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + const KIB: u64 = 1 << 10; + const MIB: u64 = 1 << 20; + match self.0 { + bytes if bytes >= MIB && bytes % MIB == 0 => write!(f, "{} MiB", bytes / MIB), + bytes if bytes >= KIB && bytes % KIB == 0 => write!(f, "{} KiB", bytes / KIB), + bytes => write!(f, "{bytes} B"), + } + } } #[derive(Debug, Clone, Deserialize)] @@ -152,6 +487,43 @@ pub struct PortPolicyFetchConfig { pub backoff_max: Duration, } +/// Application-level health polling of registered CVMs. +/// +/// The gateway asks each guest agent whether the app is serving, rather than +/// having the CVM push the answer. A wedged agent then shows up as a failed +/// poll instead of as silence that has to be told apart from "nothing changed", +/// and only CVMs that asked to be gated (`RegisterCvmRequest.health_check`) are +/// polled at all. +#[derive(Debug, Clone, Deserialize)] +pub struct HealthCheckConfig { + /// Poll instances at all. Turning this off leaves every instance eligible, + /// which is how the gateway behaved before health polling existed. + pub enabled: bool, + /// Delay between polling rounds. + /// + /// There is little point going much below this: end-to-end detection is + /// dominated by the app's own `healthcheck` settings, where Docker's + /// defaults (30s interval, 3 retries) take up to 90s to mark a container + /// unhealthy in the first place. + #[serde(with = "serde_duration")] + pub interval: Duration, + /// Timeout for a single `Worker.Health` RPC. A poll that times out counts + /// as unhealthy -- an agent that cannot answer cannot vouch for the app. + #[serde(with = "serde_duration")] + pub timeout: Duration, + /// How many instances to poll at once, so a large fleet does not arrive as + /// one burst of connections. + pub concurrency: usize, + /// Consecutive failures to *reach* an agent before a healthy instance is + /// demoted. + /// + /// Only unreachability is counted. An agent that answers "unhealthy" is + /// believed on the spot -- it can see the app and this gateway cannot. + /// Without this, one dropped packet ejects an instance and recomputes the + /// whole app's selection, and it costs a second invalidation coming back. + pub failure_threshold: u32, +} + #[derive(Debug, Clone, Deserialize)] pub struct Timeouts { #[serde(with = "serde_duration")] @@ -163,6 +535,9 @@ pub struct Timeouts { #[serde(with = "serde_duration")] pub cache_top_n: Duration, + /// Maximum WireGuard handshake age for an instance to be considered healthy. + #[serde(default = "default_handshake_stale", with = "serde_duration")] + pub handshake_stale: Duration, /// Timeout for DNS TXT record resolution (app address lookup). #[serde(with = "serde_duration")] @@ -171,6 +546,13 @@ pub struct Timeouts { pub data_timeout_enabled: bool, #[serde(with = "serde_duration")] pub idle: Duration, + /// No longer read. The per-operation write timer was replaced by the + /// connection-level progress watchdog in `io_bridge`, which catches a + /// stalled write through `idle` instead: a write that makes no progress + /// stops bumping the direction's progress counter, and the watchdog fires. + /// The key is still accepted so existing configs -- and the CVM app + /// entrypoint's `TIMEOUT_WRITE` -- keep parsing. + #[allow(dead_code)] #[serde(with = "serde_duration")] pub write: Duration, #[serde(with = "serde_duration")] @@ -208,6 +590,27 @@ pub struct SyncConfig { /// Interval for periodic WAL persistence (default: 10s) #[serde(with = "serde_duration")] pub persist_interval: Duration, + /// How long a KV write may sit in the page cache before the write-ahead log + /// is forced to disk. + /// + /// Zero means *no window*, which is the strictest setting and not the + /// cheapest one: every write is forced before it returns. It reads the + /// opposite way round from the `*_interval` fields either side of it, where + /// zero turns the work off -- which is why this one is a window. + #[serde(with = "serde_duration")] + pub wal_sync_window: Duration, + /// Collect tombstones once every this many replicated writes, per store. + /// Zero disables collection, which keeps every delete on disk for the life + /// of the deployment. + /// + /// The trigger is a count of replicated writes rather than a clock so that + /// every node in a cluster collects in the same window without agreeing on + /// the time: see `start_tombstone_gc_task`. This value is only the default + /// -- an operator override stored in the KV itself (`SetTombstoneGcConfig`) + /// replicates to every node and takes precedence, which is also the only + /// way to keep the boundaries identical when config files drift. + #[serde(default = "default_tombstone_gc_writes")] + pub tombstone_gc_writes: u64, /// Enable periodic sync of instance connections to KV store pub sync_connections_enabled: bool, /// Interval for syncing instance connections to KV store @@ -219,11 +622,11 @@ pub struct SyncConfig { pub struct Config { pub wg: WgConfig, pub proxy: ProxyConfig, - pub pccs_url: Option<String>, + #[serde(default)] + pub attestation: AttestationVerifierConfig, pub recycle: RecycleConfig, pub set_ulimit: bool, pub rpc_domain: String, - pub kms_url: String, pub admin: AdminConfig, /// Debug server configuration (separate port for debug RPCs) pub debug: DebugConfig, @@ -236,11 +639,19 @@ pub struct DebugConfig { /// Enable debug server #[serde(default)] pub insecure_enable_debug_rpc: bool, + /// Let the app-address `localhost` resolve to 127.0.0.1, so a hostname can + /// be routed to a service on the gateway host itself. + /// + /// This lives under `debug` and carries the `insecure_` prefix because the + /// app address is not only read from the platform's own `<id>.<base_domain>` + /// grammar: it also comes from the `_dstack-app-address` TXT record of an + /// arbitrary custom domain. With this on, anyone who controls any DNS zone + /// can point the gateway at its own loopback -- where the admin and debug + /// listeners bind precisely because being unreachable is their access + /// control -- and pick the port, since the `localhost` shortcut is not a + /// registered instance and so bypasses `port_policy` entirely. #[serde(default)] - pub insecure_skip_attestation: bool, - /// Path to pre-generated debug key data file (JSON format containing key, quote, event_log, and vm_config) - #[serde(default)] - pub key_file: String, + pub insecure_localhost_backend: bool, } #[derive(Debug, Clone, Deserialize)] @@ -296,8 +707,14 @@ pub struct AdminConfig { /// Shared secret required to call any admin endpoint (RPC + dashboard). /// Can also be supplied via `DSTACK_GATEWAY_ADMIN_TOKEN` / `ADMIN_API_TOKEN` /// env vars. Required unless `insecure_no_auth = true`. + /// + /// Accepts the legacy `admin_token` key for backward compatibility. + #[serde(default, alias = "admin_token")] + pub auth_token: String, + /// Optional Apache htpasswd file. Enables standard HTTP Basic auth while + /// preserving token authentication for existing clients. #[serde(default)] - pub admin_token: String, + pub htpasswd_file: PathBuf, /// Disable authentication entirely. Development/testing only; never enable /// on an admin interface that is reachable from the network. #[serde(default)] @@ -350,8 +767,25 @@ pub fn setup_wireguard(config: &WgConfig) -> Result<()> { #[cfg(test)] mod tests { use super::*; + use rocket::figment::providers::{Format, Toml}; use std::str::FromStr; + #[test] + fn admin_auth_token_reads_new_and_legacy_keys() { + // new key + let cfg: AdminConfig = + Figment::from(Toml::string("enabled = true\nauth_token = \"new\"\n")) + .extract() + .unwrap(); + assert_eq!(cfg.auth_token, "new"); + // legacy `admin_token` key still deserializes via the serde alias + let cfg: AdminConfig = + Figment::from(Toml::string("enabled = true\nadmin_token = \"legacy\"\n")) + .extract() + .unwrap(); + assert_eq!(cfg.auth_token, "legacy"); + } + #[test] fn test_validate() { // Valid configuration @@ -406,4 +840,71 @@ mod tests { "Client IP range is not in the network" ); } + + fn engage_after(toml: &str) -> EngageAfter { + Figment::from(Toml::string(toml)) + .extract() + .expect("valid EngageAfter") + } + + #[test] + fn no_gate_engages_immediately() { + let gate = engage_after(""); + assert!(gate.is_immediate()); + assert!(gate.after_bytes.is_none()); + assert!(gate.after_duration.is_none()); + } + + #[test] + fn a_configured_gate_is_not_immediate() { + assert!(!engage_after("after_bytes = 65536").is_immediate()); + assert!(!engage_after("after_duration = \"5s\"").is_immediate()); + } + + #[test] + fn byte_gate_ignores_elapsed_time() { + let gate = engage_after("after_bytes = 1024"); + let long_ago = Instant::now() - Duration::from_secs(3600); + assert!(!gate.reached(1023, long_ago)); + assert!(gate.reached(1024, long_ago)); + } + + #[test] + fn duration_gate_ignores_bytes() { + let gate = engage_after("after_duration = \"5s\""); + assert!(!gate.reached(u64::MAX, Instant::now())); + assert!(gate.reached(0, Instant::now() - Duration::from_secs(5))); + } + + #[test] + fn gates_are_independent_and_either_fires() { + // The case the byte gate alone cannot express: a low-rate stream that + // stays well under `after_bytes` but outlives `after_duration`. + let gate = engage_after("after_bytes = 65536\nafter_duration = \"5s\""); + let just_started = Instant::now(); + assert!(!gate.reached(64, just_started)); + assert!(gate.reached(65536, just_started)); + assert!(gate.reached(64, Instant::now() - Duration::from_secs(5))); + } + + #[test] + fn splice_section_is_optional() { + #[derive(Deserialize)] + struct Holder { + #[serde(default)] + tcp_splice: Option<EngageAfter>, + } + let absent: Holder = Figment::from(Toml::string("")).extract().unwrap(); + assert!( + absent.tcp_splice.is_none(), + "absent section disables splice" + ); + + let present: Holder = Figment::from(Toml::string("[tcp_splice]\nafter_duration = \"5s\"")) + .extract() + .unwrap(); + let gate = present.tcp_splice.expect("section present"); + assert_eq!(gate.after_duration, Some(Duration::from_secs(5))); + assert!(gate.after_bytes.is_none()); + } } diff --git a/dstack/gateway/src/debug_service.rs b/dstack/gateway/src/debug_service.rs index b00176a14..335d3b61f 100644 --- a/dstack/gateway/src/debug_service.rs +++ b/dstack/gateway/src/debug_service.rs @@ -36,7 +36,14 @@ impl DebugRpc for DebugRpcHandler { &request.instance_id, &request.client_public_key, "", - None, + // Reports nothing about itself: no port policy, and no statement + // either way about health gating. A brand-new debug-registered + // instance is therefore never polled and always counts as healthy, + // which keeps this path usable for testing routing without + // standing up a guest agent -- and re-registering an id that a real + // CVM already claimed leaves that CVM's declaration alone rather + // than silently downgrading it out of polling for good. + Default::default(), ) } @@ -85,6 +92,7 @@ impl DebugRpc for DebugRpcHandler { // Get all instances let instances: Vec<InstanceEntry> = kv_store .load_all_instances() + .decoded .into_iter() .map(|(instance_id, data)| InstanceEntry { instance_id, @@ -117,11 +125,7 @@ impl DebugRpc for DebugRpcHandler { .instances .values() .map(|inst| { - let reg_time = inst - .reg_time - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); + let reg_time = crate::time::encode_ts(inst.reg_time); ProxyStateInstance { instance_id: inst.id.clone(), app_id: inst.app_id.clone(), diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs index cbb316ea4..be57011b8 100644 --- a/dstack/gateway/src/distributed_certbot.rs +++ b/dstack/gateway/src/distributed_certbot.rs @@ -8,46 +8,459 @@ //! with dynamic DNS credential configuration and attestation storage. use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::Duration; use anyhow::{bail, Context, Result}; -use certbot::{AcmeClient, Dns01Client}; -use dstack_guest_agent_rpc::RawQuoteArgs; +use certbot::{ + advisory_dns_wait, resolve_issuer_domain_name, AcmeAccount, AcmeClient, ChallengeKind, + Dns01Client, ValidationMethod, +}; +use dstack_guest_agent_rpc::v0::RawQuoteArgs; use ra_tls::attestation::QuoteContentType; use ra_tls::rcgen::KeyPair; use tracing::{error, info, warn}; use crate::cert_store::CertResolver; use crate::kv::{ - AcmeAttestation, CertAttestation, CertCredentials, CertData, DnsProvider, KvStore, - ZtDomainConfig, + AcmeAttestation, CertAttestation, CertCredentials, CertData, DnsCredential, DnsProvider, + KvStore, PersistentWriteNotifier, ZtDomainConfig, }; +use crate::time::now_secs; /// Lock timeout for certificate renewal (10 minutes) const RENEW_LOCK_TIMEOUT_SECS: u64 = 600; +/// Lock timeout for ACME credential rotation (10 minutes) +const ROTATION_LOCK_TIMEOUT_SECS: u64 = 600; + /// Default ACME URL (Let's Encrypt production) const DEFAULT_ACME_URL: &str = "https://acme-v02.api.letsencrypt.org/directory"; +/// How long a `dns-persist-01` domain waits for its record to become visible. +/// +/// A `dns-01` domain reads this from its DNS credential; a `dns-persist-01` +/// domain has none, so it starts from the same 300s a credential defaults to +/// and is clamped by [`advisory_dns_wait`] like every other domain. +const DNS_PERSIST_MAX_DNS_WAIT: Duration = Duration::from_secs(300); + +/// What an ACME credential rotation left behind. +#[derive(Debug)] +pub struct RotationOutcome { + /// URI of the new ACME account. + pub account_uri: String, + /// Domains whose CAA could not be re-pinned to the new account. + pub repin_failed: Vec<String>, + /// Domains whose CAA records were re-pinned to the new account. + pub domains_updated: usize, + /// Records an operator has to publish by hand, for domains the gateway + /// cannot write. Non-empty means issuance for those domains is broken until + /// they are published. + pub required_dns_records: Vec<String>, +} + +/// What the CAA re-pin pass over a rotation's domains produced. +/// +/// The three numbers a rotation reports are all derived from one tally, so they +/// cannot disagree: `total` counts every domain the rotation covered, `manual` +/// the ones a re-pin was never attempted for -- a `dns-persist-01` domain has +/// no CAA the gateway can write -- and `failed` the ones that were tried and +/// did not take. +#[derive(Debug, Default)] +struct RepinTally { + /// Domains skipped because the gateway cannot write their DNS. + manual: usize, + /// Domains a re-pin was attempted for and failed. + failed: Vec<String>, +} + +impl RepinTally { + /// Domains a re-pin was actually attempted for. + /// + /// The denominator a failure count belongs over. Counting the manual ones + /// in reports them as successes: "2/5 failed" says three domains were + /// re-pinned when the other three were never candidates. + fn attempted(&self, total: usize) -> usize { + total.saturating_sub(self.manual) + } + + /// Domains now pinned to the new account. + fn updated(&self, total: usize) -> usize { + self.attempted(total).saturating_sub(self.failed.len()) + } +} + +/// How one ZT domain answers ACME challenges. +struct DomainValidation { + method: ValidationMethod, + /// How long to wait for the challenge records to become visible. + max_dns_wait: Duration, +} + +impl DomainValidation { + /// Whether the gateway can write this domain's DNS records itself. + fn writes_dns(&self) -> bool { + matches!(self.method, ValidationMethod::Dns01 { .. }) + } +} + /// Multi-domain certificate manager pub struct DistributedCertBot { kv_store: Arc<KvStore>, cert_resolver: Arc<CertResolver>, + write_notifier: Option<Arc<dyn PersistentWriteNotifier>>, } impl DistributedCertBot { - pub fn new(kv_store: Arc<KvStore>, cert_resolver: Arc<CertResolver>) -> Self { + pub fn new( + kv_store: Arc<KvStore>, + cert_resolver: Arc<CertResolver>, + write_notifier: Option<Arc<dyn PersistentWriteNotifier>>, + ) -> Self { Self { kv_store, cert_resolver, + write_notifier, } } - /// Get the current certbot configuration from KV store - fn config(&self) -> crate::kv::GlobalCertbotConfig { + fn notify_lock_write(&self) { + if let Some(notifier) = &self.write_notifier { + notifier.notify_persistent_write(); + } + } + + /// Acquire the lock that guards the cluster's shared ACME account. + /// + /// One lock covers every operation over that account -- rotation, CAA + /// reconciliation, and first-use registration -- because they all read or + /// re-pin the same thing, and the record they publish is last-writer-wins + /// with no compare-and-swap. It lives in WaveKV rather than in this + /// process, so it orders two nodes as well as two tasks; see + /// [`KvStore::try_acquire_rotation_lock`] for its best-effort limits. + /// + /// The lock is not reentrant, by design: a stale holder must expire rather + /// than be re-entered. A caller that holds it across a sequence of + /// per-domain work must therefore make sure nothing inside that sequence + /// takes it again -- see [`Self::ensure_acme_account`]. + fn acquire_acme_lock(&self, operation: &str) -> Result<crate::kv::CertRenewLock> { + self.try_acquire_rotation_lock().with_context(|| { + format!( + "cannot {operation}: another rotation, CAA reconciliation, or ACME account \ + registration holds the shared ACME lock; retry after it finishes" + ) + }) + } + + fn try_acquire_rotation_lock(&self) -> Option<crate::kv::CertRenewLock> { + let lock = self + .kv_store + .try_acquire_rotation_lock(ROTATION_LOCK_TIMEOUT_SECS)?; + self.notify_lock_write(); + Some(lock) + } + + fn release_rotation_lock(&self, lock: &crate::kv::CertRenewLock) -> Result<()> { + self.kv_store.release_rotation_lock(lock)?; + self.notify_lock_write(); + Ok(()) + } + + fn try_acquire_cert_lock(&self, domain: &str) -> bool { + let acquired = self + .kv_store + .try_acquire_cert_lock(domain, RENEW_LOCK_TIMEOUT_SECS); + if acquired { + self.notify_lock_write(); + } + acquired + } + + fn release_cert_lock(&self, domain: &str) -> Result<()> { + self.kv_store.release_cert_lock(domain)?; + self.notify_lock_write(); + Ok(()) + } + + async fn dns_client(&self, domain: &str, dns_cred: &DnsCredential) -> Result<Dns01Client> { + match &dns_cred.provider { + DnsProvider::Cloudflare { api_token, api_url } => { + Dns01Client::new_cloudflare(domain.to_string(), api_token.clone(), api_url.clone()) + .await + } + } + } + + /// Build the ACME validation method for one ZT domain. + /// + /// `dns-01` resolves the zone through the provider API, so a missing or + /// broken credential surfaces here rather than at the first order. + /// `dns-persist-01` reads no credential at all: the gateway CVM holds no + /// DNS write access for that domain, which is the point of the method. + async fn validation_for( + &self, + domain: &str, + config: &ZtDomainConfig, + ) -> Result<DomainValidation> { + let renew_timeout = self.config()?.renew_timeout; + match config.challenge { + ChallengeKind::Dns01 => { + let dns_cred = dns_credential_for(&self.kv_store, config)?; + let client = self.dns_client(domain, &dns_cred).await?; + Ok(DomainValidation { + method: ValidationMethod::Dns01 { + client, + txt_ttl: dns_cred.dns_txt_ttl, + issuer_domain_name: self.issuer_domain_name()?, + }, + max_dns_wait: advisory_dns_wait(dns_cred.max_dns_wait, renew_timeout), + }) + } + ChallengeKind::DnsPersist01 => Ok(DomainValidation { + method: ValidationMethod::DnsPersist01 { + issuer_domain_name: self.issuer_domain_name()?, + }, + max_dns_wait: advisory_dns_wait(DNS_PERSIST_MAX_DNS_WAIT, renew_timeout), + }), + } + } + + /// Issuer Domain Name to name in `dns-persist-01` records and in the CAA + /// records written for either challenge. + fn issuer_domain_name(&self) -> Result<String> { + resolve_issuer_domain_name(&self.config()?.issuer_domain_name) + .context("invalid issuer_domain_name in the certbot config") + } + + /// The DNS records a ZT domain's zone has to contain, as zone-file lines. + /// + /// Rendered from the stored account URI with no ACME round trip, so the + /// admin listing endpoints stay cheap. Empty until an ACME account exists — + /// every record names one — and empty rather than an error when the stored + /// credentials cannot be read, since listing a domain must not depend on + /// them. + pub fn required_dns_records(&self, config: &ZtDomainConfig) -> Vec<String> { + let Ok(issuer_domain_name) = self.issuer_domain_name() else { + return Vec::new(); + }; + let Some(account_uri) = self + .kv_store + .get_acme_credentials() + .ok() + .flatten() + .and_then(|creds| extract_account_uri(&creds.acme_credentials)) + else { + return Vec::new(); + }; + // The gateway only ever orders `*.{domain}`, so that is the whole set of + // identifiers the CA will look records up for. + certbot::required_dns_records( + config.challenge, + &issuer_domain_name, + &account_uri, + &[format!("*.{}", config.domain)], + ) + .iter() + .map(ToString::to_string) + .collect() + } + + /// Log the records an operator has to publish for a domain, and return them. + fn report_manual_records(&self, config: &ZtDomainConfig) -> Vec<String> { + let records = self.required_dns_records(config); + for record in &records { + warn!( + "cert[{}]: publish this record by hand: {record}", + config.domain + ); + } + records + } + + /// Rotate the shared ACME account without interrupting certificate serving. + /// + /// The sequence is: validate every domain's DNS credential, register the + /// replacement account, publish the new credentials, then re-pin every + /// domain's CAA record to the new account. + /// + /// A cluster with no ZT domain registers an account and stops there, which + /// is how a `dns-persist-01` deployment bootstraps: the record an operator + /// publishes names the account, so there is nothing to publish -- and + /// therefore nothing that can be issued -- until one exists. Publishing before re-pinning + /// makes the failure mode convergent: if some domains fail to re-pin, the + /// cluster is already on the new account and rerunning `SetCaa` finishes + /// the switch without registering yet another account (Let's Encrypt caps + /// new registrations per IP). + /// + /// Between publishing and re-pinning, a renewal on another node may pick up + /// the new account while a domain's CAA still pins the old one; that + /// issuance fails and the periodic renewal task retries. Re-pinning briefly + /// installs `;` guard CAA records, so a failure can leave a domain blocked + /// from issuance until a later `SetCaa` run succeeds (the same hazard as + /// [`Self::set_caa_all`]). + /// + /// This RPC re-pins issuance to the new account; it does not deactivate the + /// old ACME account at the CA. + /// + /// Runs under the shared ACME lock, so a concurrent reconciliation or + /// first-use registration on any node is refused for the duration. + pub async fn rotate_acme_credentials(&self) -> Result<RotationOutcome> { + let rotation_lock = self.acquire_acme_lock("rotate ACME credentials")?; + let result = self.do_rotate_acme_credentials().await; + if let Err(err) = self.release_rotation_lock(&rotation_lock) { + error!("failed to release ACME rotation lock: {err:?}"); + } + result + } + + async fn do_rotate_acme_credentials(&self) -> Result<RotationOutcome> { + let configs = self.kv_store.list_zt_domain_configs(); + let certbot_config = self.config()?; + let acme_url = if certbot_config.acme_url.is_empty() { + DEFAULT_ACME_URL + } else { + &certbot_config.acme_url + }; + + // Validate every domain's DNS credential up front: constructing a DNS + // client resolves the zone through an authenticated API call, so a + // misconfigured domain aborts the rotation here with no side effects + // and no ACME account consumed. A dns-persist-01 domain has no + // credential to check and is prepared without touching any provider. + let mut prepared = Vec::with_capacity(configs.len()); + for config in &configs { + let validation = self + .validation_for(&config.domain, config) + .await + .with_context(|| format!("DNS credential check failed for {}", config.domain))?; + prepared.push((config, validation)); + } + let total = prepared.len(); + + // Registration needs no domain: it proves nothing about one. A cluster + // with no ZT domain yet registers here and gets an account URI, which is + // what a `dns-persist-01` record has to name before the domain it + // authorizes can ever be issued. + let account = AcmeClient::register_account(acme_url) + .await + .context("failed to create replacement ACME account")?; + let AcmeAccount { + credentials, + account_uri, + } = account; + + // Publish immediately. From here the cluster converges on the new + // account, and recovering from a partial re-pin below never needs to + // register another account. Readers create an ACME client per + // operation, so all nodes pick this up after WaveKV propagates it. + self.kv_store.save_acme_credentials(&CertCredentials { + acme_credentials: credentials.clone(), + })?; + + // Re-pin every domain's CAA to the new account, best effort across all + // domains: one failing domain must not block re-pinning the rest. The + // first domain reuses the registration client, which is already bound + // to its validation method and the new credentials. + // + // A dns-persist-01 domain has nothing to re-pin from here, and rotation + // is not transparent for it either: its `_validation-persist` record + // still names the *old* account, so orders for that domain fail until + // the operator republishes. Those records are reported below. + let mut tally = RepinTally::default(); + let repin = |tally: &mut RepinTally, domain: &str, result: Result<()>| match result { + Ok(()) => info!("cert[{domain}]: CAA re-pinned to {account_uri}"), + Err(err) => { + error!("cert[{domain}]: failed to re-pin CAA: {err:?}"); + tally.failed.push(domain.to_string()); + } + }; + for (config, validation) in prepared { + if !validation.writes_dns() { + tally.manual += 1; + continue; + } + let result = async { + let client = + AcmeClient::load(validation.method, &credentials, validation.max_dns_wait) + .await + .context("failed to prepare ACME client")?; + client + .set_caa_records(std::slice::from_ref(&config.domain)) + .await + .context("failed to update CAA records") + } + .await; + repin(&mut tally, &config.domain, result); + } + + // Rendered after the new credentials are published, so the records name + // the account the cluster has actually switched to. + let manual = configs + .iter() + .filter(|config| config.challenge != ChallengeKind::Dns01) + .flat_map(|config| self.report_manual_records(config)) + .collect::<Vec<_>>(); + + // Attest the new account only after CAA re-pinning: attestation does + // not gate issuance, so its agent round trips must not widen the + // window where the published account and the CAA records disagree. + // Run it even when some domains failed so the new account is still + // recorded. + if let Err(err) = self.generate_and_save_acme_attestation(&account_uri).await { + warn!("failed to attest rotated ACME account: {err:?}"); + } + + if !tally.failed.is_empty() { + // Reported, not raised. The account is registered and the cluster is + // already using it, so an error here reads as "nothing happened" and + // invites a retry -- which would register yet another account + // against a rate-limited quota and rewrite CAA a second time. What + // is left is a `SetCaa` run, and the caller is told exactly that. + // + // Out of the domains a re-pin was attempted for, not out of every + // domain: a dns-persist-01 domain was never a candidate, and + // counting it in the denominator reports it as re-pinned. + error!( + "rotated to {account_uri} and published the new credentials, but failed to \ + re-pin CAA for {}/{} domains: {}; rerun SetCaa until it succeeds — \ + retrying the rotation would register yet another account", + tally.failed.len(), + tally.attempted(total), + tally.failed.join(", ") + ); + } + if tally.manual > 0 { + warn!( + "{}/{total} domains use dns-persist-01: issuance for them stays \ + broken until the records above name the new account {account_uri}", + tally.manual + ); + } + Ok(RotationOutcome { + account_uri, + domains_updated: tally.updated(total), + required_dns_records: manual, + repin_failed: tally.failed, + }) + } + + /// Get the current certbot configuration from KV store. + /// + /// Propagates a corrupt record instead of falling back to the defaults: + /// the default `acme_url` is Let's Encrypt production, so a silent + /// fallback would move issuance to a different ACME server. + fn config(&self) -> Result<crate::kv::GlobalCertbotConfig> { self.kv_store.get_certbot_config() } + /// The ACME directory this cluster issues from. + fn acme_url(&self) -> Result<String> { + let config = self.config()?; + if config.acme_url.is_empty() { + return Ok(DEFAULT_ACME_URL.to_string()); + } + Ok(config.acme_url) + } + /// Initialize all ZT-Domain certificates pub async fn init_all(&self) -> Result<()> { let configs = self.kv_store.list_zt_domain_configs(); @@ -82,6 +495,137 @@ impl DistributedCertBot { self.request_new_cert(domain).await } + /// Set CAA records for every configured ZT domain. + /// + /// Runs under the shared ACME lock, so a rotation or another + /// reconciliation -- on this node or any other -- is refused for the + /// duration rather than interleaved with it. Reconciling a domain rewrites + /// its issuer records in place (guard, sweep, write, unguard), and two runs + /// over the same zone delete each other's records. + /// + /// Reconciliation is per-domain best effort: a failing domain is logged and the + /// remaining domains are still reconciled, so one misconfigured domain cannot + /// leave the rest unauthorized. Failures are reported together in the returned + /// error. + /// + /// Note that reconciling a domain briefly installs `;` guard CAA records that + /// forbid issuance for that name. A failure in the middle of the sequence can + /// leave those guards behind, blocking issuance for the domain until a later run + /// succeeds. The guard window can also fail an ACME order that is in flight for + /// the same domain; the periodic renewal task retries, so that is transient. + pub async fn set_caa_all(&self) -> Result<()> { + let configs = self.kv_store.list_zt_domain_configs(); + if configs.is_empty() { + warn!("no ZT-Domain configured, no CAA records to set"); + return Ok(()); + } + // Before the lock, never under it: registration takes the same lock. + self.ensure_acme_account(&configs).await?; + let rotation_lock = self.acquire_acme_lock("reconcile CAA records")?; + let result = self.do_set_caa_all(configs).await; + if let Err(err) = self.release_rotation_lock(&rotation_lock) { + error!("failed to release ACME rotation lock: {err:?}"); + } + result + } + + /// Register the cluster's shared ACME account if it does not exist yet. + /// + /// Called before taking the ACME lock, by callers that hold it across + /// per-domain work that would otherwise register lazily from inside the + /// locked region and be refused by their own lock. The record is read + /// first, so the steady state -- an account already registered -- costs one + /// KV read and no provider or CA round trip. + /// + /// Any configured domain will do: the account is shared, and registration + /// only needs one domain's DNS credential to build a client, the same way + /// rotation uses the first domain's. + async fn ensure_acme_account(&self, configs: &[ZtDomainConfig]) -> Result<()> { + let stored = self + .kv_store + .get_acme_credentials() + .context("call RotateAcmeCredentials to replace the stored ACME credentials")?; + if stored.is_some() { + return Ok(()); + } + let Some(config) = configs.first() else { + return Ok(()); + }; + self.get_or_create_acme_client(&config.domain, config) + .await + .context("failed to register the shared ACME account")?; + Ok(()) + } + + async fn do_set_caa_all(&self, configs: Vec<ZtDomainConfig>) -> Result<()> { + let total = configs.len(); + let mut failed = Vec::new(); + let mut skipped = 0usize; + for config in configs { + let domain = config.domain.clone(); + // A dns-persist-01 domain has no DNS write access to reconcile with. + // Log the records the operator owns instead of failing the call, so + // one such domain does not make SetCaa unusable for the rest. + if config.challenge != ChallengeKind::Dns01 { + self.report_manual_records(&config); + skipped += 1; + continue; + } + match self.set_caa(&domain, &config).await { + Ok(()) => info!("cert[{domain}]: CAA records reconciled"), + Err(err) => { + error!("cert[{domain}]: failed to set CAA records: {err:?}"); + failed.push(domain); + } + } + } + + if !failed.is_empty() { + // Out of the domains actually attempted. Counting the skipped + // dns-persist-01 ones in the denominator would read as though they + // had succeeded, which is the same miscount `RepinTally::attempted` + // exists to avoid on the rotation path. + bail!( + "failed to set CAA records for {}/{} domains: {}; \ + they may retain guard CAA records that block issuance until a rerun succeeds", + failed.len(), + total - skipped, + failed.join(", ") + ); + } + info!( + "CAA records reconciled for {} domains ({skipped} on dns-persist-01 left to the operator)", + total - skipped + ); + Ok(()) + } + + /// Set CAA records for a single ZT domain. + /// + /// The domain in the config is the base domain and certificates are issued for + /// `*.{domain}`, which the CAA lookup covers by climbing to the base domain. + /// + /// The written CAA value pins `accounturi` to the global ACME account, which + /// the caller has already registered; see [`Self::ensure_acme_account`]. + async fn set_caa(&self, domain: &str, config: &ZtDomainConfig) -> Result<()> { + // Load only. This runs under the ACME lock, which is not reentrant, so + // registering here would refuse the run that took it -- and the caller + // has already registered the account before taking it. Nothing under + // the lock may call [`Self::acquire_acme_lock`], and keeping the + // registering variant out of this path is what makes that structural + // rather than a rule to remember. + let acme_url = self.acme_url()?; + let acme_client = self + .load_stored_acme_client(domain, config, &acme_url) + .await + .context("failed to initialize ACME client")? + .context("no shared ACME account is registered for this cluster")?; + acme_client + .set_caa_records(&[domain.to_string()]) + .await + .context("failed to set CAA records") + } + /// Try to renew all ZT-Domain certificates pub async fn try_renew_all(&self) -> Result<()> { let configs = self.kv_store.list_zt_domain_configs(); @@ -109,7 +653,7 @@ impl DistributedCertBot { } else if let Some(ref data) = cert_data { let now = now_secs(); let expires_in = data.not_after.saturating_sub(now); - expires_in < self.config().renew_before_expiration.as_secs() + expires_in < self.config()?.renew_before_expiration.as_secs() } else { true }; @@ -120,10 +664,7 @@ impl DistributedCertBot { } // Try to acquire lock - if !self - .kv_store - .try_acquire_cert_lock(domain, RENEW_LOCK_TIMEOUT_SECS) - { + if !self.try_acquire_cert_lock(domain) { info!("another node is renewing, skipping"); return Ok(false); } @@ -140,7 +681,7 @@ impl DistributedCertBot { }; // Release lock regardless of result - if let Err(err) = self.kv_store.release_cert_lock(domain) { + if let Err(err) = self.release_cert_lock(domain) { error!("failed to release lock: {err:?}"); } @@ -156,10 +697,7 @@ impl DistributedCertBot { .context("ZT-Domain config not found")?; // Try to acquire lock first - if !self - .kv_store - .try_acquire_cert_lock(domain, RENEW_LOCK_TIMEOUT_SECS) - { + if !self.try_acquire_cert_lock(domain) { // Another node is requesting, wait for it info!("another node is requesting, waiting..."); tokio::time::sleep(Duration::from_secs(30)).await; @@ -172,7 +710,7 @@ impl DistributedCertBot { let result = self.do_request_new(domain, &config).await; - if let Err(err) = self.kv_store.release_cert_lock(domain) { + if let Err(err) = self.release_cert_lock(domain) { error!("failed to release lock: {err:?}"); } @@ -194,7 +732,7 @@ impl DistributedCertBot { wildcard_domain ); let cert_pem = tokio::time::timeout( - self.config().renew_timeout, + self.config()?.renew_timeout, acme_client.request_new_certificate(&key_pem, &[wildcard_domain]), ) .await @@ -248,7 +786,7 @@ impl DistributedCertBot { wildcard_domain ); let new_cert_pem = tokio::time::timeout( - self.config().renew_timeout, + self.config()?.renew_timeout, // Note: we request a new cert rather than renew, since we have a new key acme_client.request_new_certificate(&key_pem, &[wildcard_domain]), ) @@ -289,59 +827,101 @@ impl DistributedCertBot { domain: &str, config: &ZtDomainConfig, ) -> Result<AcmeClient> { - // Get DNS credential (from config or default) - let dns_cred = if let Some(ref cred_id) = config.dns_cred_id { - self.kv_store - .get_dns_credential(cred_id) - .context("specified DNS credential not found")? - } else { - self.kv_store - .get_default_dns_credential() - .context("no default DNS credential configured")? - }; + let acme_url = self.acme_url()?; - // Create DNS client based on provider - let dns01_client = match &dns_cred.provider { - DnsProvider::Cloudflare { api_token, api_url } => { - Dns01Client::new_cloudflare(domain.to_string(), api_token.clone(), api_url.clone()) - .await? - } - }; + if let Some(client) = self + .load_stored_acme_client(domain, config, &acme_url) + .await? + { + info!("loaded global ACME account credentials from KvStore"); + return Ok(client); + } - // Use ACME URL from certbot config, fall back to default if not set - let config = self.config(); - let acme_url = if config.acme_url.is_empty() { - DEFAULT_ACME_URL - } else { - &config.acme_url + // Registering is the one step in this function that cannot be repeated + // harmlessly. Each run spends a rate-limited registration at the CA, and + // the credentials record is last-writer-wins with no compare-and-swap, + // so a concurrent registration's account is simply dropped -- while the + // attestation written beside it, under its own key with its own + // last-writer-wins race, may well be the one that survives. Renewal + // locks are per domain and do not help: a fresh cluster registers from + // however many domains and nodes start at once. Take the shared ACME + // lock and look again before spending a registration. + let rotation_lock = self.acquire_acme_lock("register the shared ACME account")?; + let client = self + .register_or_adopt_account(domain, config, &acme_url) + .await; + if let Err(err) = self.release_rotation_lock(&rotation_lock) { + error!("failed to release ACME rotation lock: {err:?}"); + } + client + } + + /// Build an ACME client from the cluster's stored account credentials, or + /// `None` if no account has been registered yet. + /// + /// A corrupt record is an error, not absence: falling through to account + /// registration would silently create an account that the account-bound CAA + /// records refuse, and burn a rate-limited registration. So is a record for + /// a different ACME directory -- registering a fresh account there would + /// leave every domain's CAA pinned to the old account and block issuance, + /// while rotation re-pins CAA along with the switch. + async fn load_stored_acme_client( + &self, + domain: &str, + config: &ZtDomainConfig, + acme_url: &str, + ) -> Result<Option<AcmeClient>> { + let Some(creds) = self + .kv_store + .get_acme_credentials() + .context("call RotateAcmeCredentials to replace the stored ACME credentials")? + else { + return Ok(None); }; + if !acme_url_matches(&creds.acme_credentials, acme_url).context( + "invalid ACME credentials in KvStore; call RotateAcmeCredentials to replace them", + )? { + bail!( + "stored ACME credentials are for a different ACME directory; \ + call RotateAcmeCredentials to switch directories" + ); + } + let validation = self.validation_for(domain, config).await?; + let client = AcmeClient::load( + validation.method, + &creds.acme_credentials, + validation.max_dns_wait, + ) + .await + .context("failed to load ACME client from KvStore credentials")?; + Ok(Some(client)) + } - // Try to load global ACME credentials from KvStore - if let Some(creds) = self.kv_store.get_acme_credentials() { - if acme_url_matches(&creds.acme_credentials, acme_url) { - info!("loaded global ACME account credentials from KvStore"); - return AcmeClient::load( - dns01_client, - &creds.acme_credentials, - dns_cred.max_dns_wait, - dns_cred.dns_txt_ttl, - ) - .await - .context("failed to load ACME client from KvStore credentials"); - } - warn!("ACME URL mismatch in KvStore credentials, will create new account"); + /// Register the cluster's shared ACME account, or adopt the one that + /// appeared while this call was waiting for the lock. + /// + /// Called with the ACME lock held. The re-read is the point: without it + /// every waiter registers an account of its own the moment it is let + /// through, which is the race the lock was taken to avoid. + async fn register_or_adopt_account( + &self, + domain: &str, + config: &ZtDomainConfig, + acme_url: &str, + ) -> Result<AcmeClient> { + if let Some(client) = self + .load_stored_acme_client(domain, config, acme_url) + .await? + { + info!("adopted the ACME account registered while this node waited for the lock"); + return Ok(client); } - // Create new global ACME account info!("creating new global ACME account at {acme_url}"); - let client = AcmeClient::new_account( - acme_url, - dns01_client, - dns_cred.max_dns_wait, - dns_cred.dns_txt_ttl, - ) - .await - .context("failed to create new ACME account")?; + let validation = self.validation_for(domain, config).await?; + let client = AcmeClient::new_account(acme_url, validation.method, validation.max_dns_wait) + .await + .context("failed to create new ACME account")?; let creds_json = client .dump_credentials() @@ -374,7 +954,8 @@ impl DistributedCertBot { .to_report_data(account_uri.as_bytes()) .to_vec(); - // Get quote + // Get quote. GetQuote is Intel TDX only, so this is best-effort: on other + // platforms the versioned attestation below is the only evidence available. let quote = match agent .get_quote(RawQuoteArgs { report_data: report_data.clone(), @@ -384,7 +965,7 @@ impl DistributedCertBot { Ok(resp) => serde_json::to_string(&resp).unwrap_or_default(), Err(err) => { warn!("failed to get TDX quote for ACME account: {err:?}"); - return Ok(()); + String::new() } }; @@ -397,6 +978,11 @@ impl DistributedCertBot { } }; + if quote.is_empty() && attestation_str.is_empty() { + warn!("no attestation evidence for ACME account, skipping save"); + return Ok(()); + } + let attestation = AcmeAttestation { account_uri: account_uri.to_string(), quote, @@ -444,7 +1030,8 @@ impl DistributedCertBot { .to_report_data(public_key_der) .to_vec(); - // Get quote + // Get quote. GetQuote is Intel TDX only, so this is best-effort: on other + // platforms the versioned attestation below is the only evidence available. let quote = match agent .get_quote(RawQuoteArgs { report_data: report_data.clone(), @@ -454,7 +1041,7 @@ impl DistributedCertBot { Ok(resp) => serde_json::to_string(&resp).unwrap_or_default(), Err(err) => { warn!(domain, "failed to generate TDX quote: {err:?}"); - return Ok(()); + String::new() } }; @@ -467,6 +1054,11 @@ impl DistributedCertBot { } }; + if quote.is_empty() && attestation.is_empty() { + warn!(domain, "no attestation evidence for cert, skipping save"); + return Ok(()); + } + let attestation = CertAttestation { public_key: public_key_der.to_vec(), quote, @@ -481,11 +1073,16 @@ impl DistributedCertBot { } } -fn now_secs() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() +fn dns_credential_for(kv_store: &KvStore, config: &ZtDomainConfig) -> Result<DnsCredential> { + if let Some(ref cred_id) = config.dns_cred_id { + kv_store + .get_dns_credential(cred_id)? + .context("specified DNS credential not found") + } else { + kv_store + .get_default_dns_credential()? + .context("no default DNS credential configured") + } } fn get_cert_expiry(cert_pem: &str) -> Option<u64> { @@ -495,19 +1092,18 @@ fn get_cert_expiry(cert_pem: &str) -> Option<u64> { Some(cert.validity().not_after.timestamp() as u64) } -fn acme_url_matches(credentials_json: &str, expected_url: &str) -> bool { +fn acme_url_matches(credentials_json: &str, expected_url: &str) -> Result<bool> { #[derive(serde::Deserialize)] struct Creds { - #[serde(default)] acme_url: String, } - serde_json::from_str::<Creds>(credentials_json) - .map(|c| c.acme_url == expected_url) - .unwrap_or(false) + let credentials = serde_json::from_str::<Creds>(credentials_json) + .context("failed to decode ACME credentials")?; + Ok(credentials.acme_url == expected_url) } /// Extract account_id (URI) from ACME credentials JSON -fn extract_account_uri(credentials_json: &str) -> Option<String> { +pub(crate) fn extract_account_uri(credentials_json: &str) -> Option<String> { #[derive(serde::Deserialize)] struct Creds { #[serde(default)] @@ -518,3 +1114,445 @@ fn extract_account_uri(credentials_json: &str) -> Option<String> { .filter(|c| !c.account_id.is_empty()) .map(|c| c.account_id) } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[derive(Default)] + struct CountingNotifier(AtomicUsize); + + impl PersistentWriteNotifier for CountingNotifier { + fn notify_persistent_write(&self) { + self.0.fetch_add(1, Ordering::Relaxed); + } + } + + fn test_certbot(data_dir: &std::path::Path) -> DistributedCertBot { + let kv_store = + Arc::new(KvStore::new(1, vec![], data_dir, None).expect("failed to create kv store")); + DistributedCertBot::new(kv_store, Arc::new(CertResolver::new()), None) + } + + /// A DNS credential pointing at a closed port: any provider API call fails + /// fast, so a test that reaches one fails instead of hanging on the network. + fn unreachable_dns_credential() -> DnsCredential { + DnsCredential { + id: "cred-1".to_string(), + name: "unreachable".to_string(), + provider: DnsProvider::Cloudflare { + api_token: "token".to_string(), + api_url: Some("http://127.0.0.1:1/client/v4".to_string()), + }, + max_dns_wait: Duration::from_secs(1), + dns_txt_ttl: 60, + created_at: 0, + updated_at: 0, + } + } + + fn test_zt_domain_config() -> ZtDomainConfig { + ZtDomainConfig { + domain: "app.example.com".to_string(), + dns_cred_id: Some("cred-1".to_string()), + port: 443, + node: None, + priority: 0, + challenge: ChallengeKind::Dns01, + } + } + + /// A certbot with one ZT domain whose DNS provider is unreachable. + fn certbot_with_domain(data_dir: &std::path::Path) -> DistributedCertBot { + let certbot = test_certbot(data_dir); + certbot + .kv_store + .save_dns_credential(&unreachable_dns_credential()) + .expect("failed to store dns credential"); + certbot + .kv_store + .save_zt_domain_config(&test_zt_domain_config()) + .expect("failed to store zt domain config"); + certbot + } + + fn save_credentials_for(certbot: &DistributedCertBot, acme_url: &str) { + certbot + .kv_store + .save_acme_credentials(&CertCredentials { + acme_credentials: format!(r#"{{"acme_url":"{acme_url}"}}"#), + }) + .expect("failed to store acme credentials"); + } + + #[test] + fn lock_writes_wake_the_persistent_push_path() { + let data_dir = tempfile::tempdir().expect("failed to create temp dir"); + let kv_store = Arc::new( + KvStore::new(1, vec![], data_dir.path(), None).expect("failed to create kv store"), + ); + let notifier = Arc::new(CountingNotifier::default()); + let certbot = DistributedCertBot::new( + kv_store, + Arc::new(CertResolver::new()), + Some(notifier.clone()), + ); + + let rotation = certbot + .try_acquire_rotation_lock() + .expect("rotation lock should be free"); + assert_eq!(notifier.0.load(Ordering::Relaxed), 1); + certbot + .release_rotation_lock(&rotation) + .expect("rotation lock release should succeed"); + assert_eq!(notifier.0.load(Ordering::Relaxed), 2); + + assert!(certbot.try_acquire_cert_lock("example.com")); + assert_eq!(notifier.0.load(Ordering::Relaxed), 3); + assert!(!certbot.try_acquire_cert_lock("example.com")); + assert_eq!( + notifier.0.load(Ordering::Relaxed), + 3, + "a rejected acquisition did not write and must not wake push" + ); + certbot + .release_cert_lock("example.com") + .expect("renewal lock release should succeed"); + assert_eq!(notifier.0.load(Ordering::Relaxed), 4); + } + + #[tokio::test] + async fn set_caa_all_succeeds_without_configured_domains() { + let data_dir = tempfile::tempdir().expect("failed to create temp dir"); + let certbot = test_certbot(data_dir.path()); + // No ZT-Domain configured: nothing to reconcile and no DNS provider is contacted. + certbot + .set_caa_all() + .await + .expect("set_caa_all should succeed without domains"); + } + + /// CAA reconciliation rewrites a zone's issuer records in place -- guard, + /// sweep, write, unguard -- so two runs over the same zone can delete each + /// other's records and leave the guards behind, which blocks issuance until + /// a later run succeeds. The lock that orders them is the shared ACME one, + /// so it orders another node's rotation against this run too. + #[tokio::test] + async fn set_caa_all_rejects_concurrent_runs() { + let data_dir = tempfile::tempdir().expect("failed to create temp dir"); + let certbot = certbot_with_domain(data_dir.path()); + // An account already exists, so this run has nothing to register and + // reaches the lock it takes for the reconciliation itself. + save_credentials_for(&certbot, DEFAULT_ACME_URL); + assert!(certbot + .kv_store + .try_acquire_rotation_lock(ROTATION_LOCK_TIMEOUT_SECS) + .is_some()); + + let err = certbot + .set_caa_all() + .await + .expect_err("a concurrent run should be rejected"); + assert!( + err.to_string().contains("cannot reconcile CAA records"), + "unexpected error: {err}" + ); + } + + /// Registration takes the same lock this run holds for the reconciliation, + /// and the lock is not reentrant. Registering up front is what keeps a + /// fresh cluster's first `SetCaa` from refusing -- or deadlocking on -- its + /// own lock. + #[tokio::test] + async fn set_caa_all_registers_the_account_before_taking_the_lock() { + let data_dir = tempfile::tempdir().expect("failed to create temp dir"); + let certbot = certbot_with_domain(data_dir.path()); + + let err = tokio::time::timeout(Duration::from_secs(30), certbot.set_caa_all()) + .await + .expect("set_caa_all must not block on its own lock") + .expect_err("the unreachable DNS provider should fail the run"); + assert!( + err.to_string() + .contains("failed to register the shared ACME account"), + "unexpected error: {err:?}" + ); + // A failed run leaves the lock free for the next attempt. + assert!(certbot + .kv_store + .try_acquire_rotation_lock(ROTATION_LOCK_TIMEOUT_SECS) + .is_some()); + } + + /// First-use registration must wait on the same lock rotation takes, or a + /// fresh cluster registers one account per node that happens to start a + /// renewal -- and the credentials record, being last-writer-wins, keeps + /// exactly one of them. + /// + /// Reaching the lock at all is the assertion: the DNS provider client is + /// built only after the lock is granted, so an unreachable provider (as + /// configured here) cannot be what this run fails on. + #[tokio::test] + async fn first_use_registration_waits_for_the_rotation_lock() { + let data_dir = tempfile::tempdir().expect("failed to create temp dir"); + let certbot = certbot_with_domain(data_dir.path()); + let config = test_zt_domain_config(); + + // Another node is mid-registration or mid-rotation. + assert!(certbot + .kv_store + .try_acquire_rotation_lock(ROTATION_LOCK_TIMEOUT_SECS) + .is_some()); + + let err = match certbot + .get_or_create_acme_client(&config.domain, &config) + .await + { + Ok(_) => panic!("registration must not proceed while the lock is held"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("cannot register the shared ACME account"), + "unexpected error: {err}" + ); + } + + /// Nothing inside the locked region may take the lock again. With an + /// account already registered, a run reaches the DNS provider -- it fails + /// there, on the reconciliation itself, and never on its own lock. + #[tokio::test] + async fn set_caa_all_does_not_take_its_own_lock_again() { + let data_dir = tempfile::tempdir().expect("failed to create temp dir"); + let certbot = certbot_with_domain(data_dir.path()); + save_credentials_for(&certbot, DEFAULT_ACME_URL); + + let err = tokio::time::timeout(Duration::from_secs(30), certbot.set_caa_all()) + .await + .expect("set_caa_all must not block on its own lock") + .expect_err("the unreachable DNS provider should fail the run"); + let msg = format!("{err:#}"); + assert!( + msg.contains("failed to set CAA records for 1/1 domains"), + "unexpected error: {msg}" + ); + assert!( + !msg.contains("shared ACME lock"), + "the run refused its own lock: {msg}" + ); + assert!(certbot + .kv_store + .try_acquire_rotation_lock(ROTATION_LOCK_TIMEOUT_SECS) + .is_some()); + } + + /// Whoever the lock lets through next must look at the record again. Its + /// contents decide the outcome -- adopt, or refuse a directory switch -- + /// and reaching either one proves no second registration was spent. + #[tokio::test] + async fn registration_rereads_the_record_under_the_lock() { + let data_dir = tempfile::tempdir().expect("failed to create temp dir"); + let certbot = certbot_with_domain(data_dir.path()); + // The account that appeared while this call waited is registered at a + // different ACME directory than the one this run is configured for. + save_credentials_for(&certbot, "https://acme.test/directory"); + + let err = match certbot + .register_or_adopt_account( + "app.example.com", + &test_zt_domain_config(), + DEFAULT_ACME_URL, + ) + .await + { + Ok(_) => panic!("a directory switch must not be made by registering"), + Err(err) => err, + }; + assert!( + err.to_string().contains("different ACME directory"), + "unexpected error: {err:?}" + ); + } + + /// Two rotations on this node are ordered by the same lock that orders two + /// nodes: the lock lives in the KV store, so a second in-process run sees + /// the first one's record. + /// The check has to finish inside the timeout wrapping the order, or the + /// "proceed anyway" exit it is designed around is never reached and the + /// renewal dies with a timeout instead of a diagnosis. Both defaults are + /// 300s, which is exactly the case that fails. + #[test] + fn the_dns_wait_ends_before_the_order_times_out() { + let default_credential = Duration::from_secs(300); + let default_renew_timeout = Duration::from_secs(300); + assert!( + advisory_dns_wait(default_credential, default_renew_timeout) < default_renew_timeout + ); + + // Lowering renew_timeout from the dashboard has to keep the invariant, + // which is why this is derived rather than a constant. + for secs in [30, 60, 120, 600] { + let renew_timeout = Duration::from_secs(secs); + assert!( + advisory_dns_wait(DNS_PERSIST_MAX_DNS_WAIT, renew_timeout) < renew_timeout, + "renew_timeout={secs}s leaves no room for the order" + ); + } + } + + /// A dns-persist-01 domain is never a re-pin candidate, so counting it into + /// the denominator reports it as re-pinned. Five domains, three of them + /// manual and the other two failing, is "2/2 failed" and zero updated -- + /// not "2/5 failed", which claims three succeeded. + #[test] + fn the_failure_count_is_out_of_the_domains_actually_tried() { + let tally = RepinTally { + manual: 3, + failed: vec!["a.example".to_string(), "b.example".to_string()], + }; + assert_eq!(tally.attempted(5), 2); + assert_eq!(tally.updated(5), 0); + } + + /// Every domain manual: nothing was attempted and nothing was updated. The + /// arithmetic has to land on zero rather than underflow, which as a usize + /// subtraction would panic the rotation after the account was published. + #[test] + fn an_all_manual_rotation_updates_nothing() { + let tally = RepinTally { + manual: 4, + failed: vec![], + }; + assert_eq!(tally.attempted(4), 0); + assert_eq!(tally.updated(4), 0); + } + + /// The ordinary case: every domain tried and every one taken. + #[test] + fn a_clean_rotation_updates_every_domain() { + let tally = RepinTally::default(); + assert_eq!(tally.attempted(3), 3); + assert_eq!(tally.updated(3), 3); + } + + /// A credential asking for less than its share keeps asking for less: the + /// clamp is a ceiling, not a target. + #[test] + fn a_shorter_configured_wait_is_left_alone() { + assert_eq!( + advisory_dns_wait(Duration::from_secs(20), Duration::from_secs(300)), + Duration::from_secs(20) + ); + } + + #[tokio::test] + async fn rotate_acme_credentials_rejects_concurrent_runs() { + let data_dir = tempfile::tempdir().expect("failed to create temp dir"); + let certbot = test_certbot(data_dir.path()); + let _held = certbot + .try_acquire_rotation_lock() + .expect("lock should be free"); + let err = certbot + .rotate_acme_credentials() + .await + .expect_err("a concurrent run should be rejected"); + assert!( + err.to_string().contains("cannot rotate ACME credentials"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn rotate_acme_credentials_rejects_when_another_node_holds_the_lock() { + let data_dir = tempfile::tempdir().expect("failed to create temp dir"); + let certbot = test_certbot(data_dir.path()); + assert!(certbot + .kv_store + .try_acquire_rotation_lock(ROTATION_LOCK_TIMEOUT_SECS) + .is_some()); + let err = certbot + .rotate_acme_credentials() + .await + .expect_err("rotation should be rejected while the KV lock is held"); + assert!( + err.to_string().contains("cannot rotate ACME credentials"), + "unexpected error: {err}" + ); + } + + /// Registering without a domain is the `dns-persist-01` bootstrap: the + /// record names the account, so the account comes first. The run here gets + /// as far as the ACME server -- an unroutable one, so the test stays + /// offline -- which is proof enough that no domain was demanded before it. + #[tokio::test] + async fn rotate_acme_credentials_registers_without_any_domain() { + let data_dir = tempfile::tempdir().expect("failed to create temp dir"); + let certbot = test_certbot(data_dir.path()); + certbot + .kv_store + .set_certbot_config(&crate::kv::GlobalCertbotConfig { + acme_url: "http://127.0.0.1:1/directory".to_string(), + ..Default::default() + }) + .expect("failed to store certbot config"); + let err = certbot + .rotate_acme_credentials() + .await + .expect_err("an unroutable ACME server cannot register an account"); + assert!( + err.to_string().contains("replacement ACME account"), + "unexpected error: {err}" + ); + // The failed rotation must release the KV lock so a later run can proceed. + assert!(certbot + .kv_store + .try_acquire_rotation_lock(ROTATION_LOCK_TIMEOUT_SECS) + .is_some()); + } + + #[tokio::test] + async fn stale_rotation_holder_does_not_release_a_newer_lock() { + let data_dir = tempfile::tempdir().expect("failed to create temp dir"); + let certbot = test_certbot(data_dir.path()); + let current = certbot + .kv_store + .try_acquire_rotation_lock(ROTATION_LOCK_TIMEOUT_SECS) + .expect("lock should be free"); + // Simulate a holder that exceeded the timeout and was superseded. + let stale = crate::kv::CertRenewLock { + started_at: current.started_at.saturating_sub(100), + started_by: 99, + }; + certbot + .kv_store + .release_rotation_lock(&stale) + .expect("stale release should be a no-op, not an error"); + assert!( + certbot.kv_store.get_rotation_lock().is_some(), + "the newer holder's lock must remain in place" + ); + certbot + .kv_store + .release_rotation_lock(¤t) + .expect("owner release should succeed"); + assert!(certbot.kv_store.get_rotation_lock().is_none()); + } + + #[test] + fn corrupt_acme_credentials_fail_closed() { + assert!(acme_url_matches("not-json", "https://acme.test/directory").is_err()); + assert!(acme_url_matches("{}", "https://acme.test/directory").is_err()); + } + + #[test] + fn valid_acme_credentials_distinguish_directory() { + let credentials = r#"{"acme_url":"https://acme.test/directory"}"#; + assert!(acme_url_matches(credentials, "https://acme.test/directory") + .expect("valid credentials rejected")); + assert!( + !acme_url_matches(credentials, "https://other.test/directory") + .expect("valid credentials rejected") + ); + } +} diff --git a/dstack/gateway/src/gen_debug_key.rs b/dstack/gateway/src/gen_debug_key.rs deleted file mode 100644 index c710548a6..000000000 --- a/dstack/gateway/src/gen_debug_key.rs +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> -// -// SPDX-License-Identifier: Apache-2.0 - -// Run with: cargo run --bin gen_debug_key -- <simulator_url> -// Example: cargo run --bin gen_debug_key -- https://daee134c3b9f66aa2401c3b5ea64f1d34038f45d-3000.tdxlab.dstack.org:12004 - -use anyhow::{Context, Result}; -use base64::{engine::general_purpose::STANDARD, Engine as _}; -use dstack_guest_agent_rpc::{dstack_guest_client::DstackGuestClient, RawQuoteArgs}; -use http_client::prpc::PrpcClient; -use ra_tls::attestation::QuoteContentType; -use ra_tls::rcgen::KeyPair; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct DebugKeyData { - /// Private key in PEM format - key_pem: String, - /// TDX quote in base64 format - quote_base64: String, - /// Event log in JSON string format - event_log: String, - /// VM config in JSON string format - vm_config: String, -} - -#[tokio::main] -async fn main() -> Result<()> { - let args: Vec<String> = std::env::args().collect(); - if args.len() != 2 { - eprintln!("Usage: {} <simulator_url>", args[0]); - eprintln!("Example: {} https://daee134c3b9f66aa2401c3b5ea64f1d34038f45d-3000.tdxlab.dstack.org:12004", args[0]); - std::process::exit(1); - } - let simulator_url = &args[1]; - - // Generate key pair - let key = KeyPair::generate().context("Failed to generate key")?; - let pubkey = key.public_key_der(); - let key_pem = key.serialize_pem(); - - // Calculate report_data - let report_data = QuoteContentType::RaTlsCert.to_report_data(&pubkey); - - // Get quote from simulator - println!("Getting quote from simulator: {simulator_url}"); - let simulator_client = PrpcClient::new(simulator_url.to_string()); - let simulator_client = DstackGuestClient::new(simulator_client); - let quote_response = simulator_client - .get_quote(RawQuoteArgs { - report_data: report_data.to_vec(), - }) - .await - .context("Failed to get quote from simulator")?; - - // Create debug key data structure - let debug_data = DebugKeyData { - key_pem, - quote_base64: STANDARD.encode("e_response.quote), - event_log: quote_response.event_log, - vm_config: quote_response.vm_config, - }; - - // Write to single JSON file - let json_content = - serde_json::to_string_pretty(&debug_data).context("Failed to serialize debug key data")?; - let output_file = "debug_key.json"; - fs_err::write(output_file, json_content).context("Failed to write debug key file")?; - - println!("✓ Successfully generated debug key data:"); - println!(" - {output_file}"); - println!("\nYou can now configure this path in your gateway config:"); - println!("[core.debug]"); - println!("insecure_skip_attestation = true"); - println!( - "key_file = \"{}\"", - fs_err::canonicalize(output_file) - .unwrap_or_default() - .display() - ); - - Ok(()) -} diff --git a/dstack/gateway/src/kv/compat.rs b/dstack/gateway/src/kv/compat.rs new file mode 100644 index 000000000..6f8ecee5d --- /dev/null +++ b/dstack/gateway/src/kv/compat.rs @@ -0,0 +1,735 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Keeps a rolling upgrade from erasing fields the writer has never heard of. +//! +//! KV values are msgpack maps keyed by field name, so a gateway running an older +//! build already *reads* a newer record without tripping over the fields it does +//! not know. The write path is where the data is lost: `encode` serializes the +//! struct as this binary declares it, so the moment an older node rewrites a +//! record — a re-registration, an admin call, a lazily-filled port policy — the +//! newer fields are gone for the whole cluster, and the node that wrote them has +//! no way to tell. +//! +//! That makes "values may gain fields freely" true only while every node is on +//! the same build, which is exactly when it does not matter. This module makes it +//! true during the upgrade window: before a record is written, the fields present +//! in the stored copy that this binary does not declare are appended to the +//! outgoing encoding verbatim. +//! +//! # What counts as unknown +//! +//! Unknown is decided against the field list `T` *declares*, obtained from the +//! derived `Deserialize` impl, not against the keys the new encoding happens to +//! contain. The difference matters: a `#[serde(skip_serializing_if)]` field that +//! this binary deliberately cleared is absent from the encoding but present in +//! the declaration, so it stays cleared instead of being resurrected from the +//! stored copy. Aliases count as declared, so `#[serde(alias)]` is safe here too. +//! +//! # The caller says whether the write is an update +//! +//! A write that updates a record outliving it — an instance, a node, a +//! credential, a config — keeps the fields it does not declare. A write that +//! states a complete new fact does not: a certificate, an attestation, a lock, +//! a counter. Carrying a field across one of those would attribute the previous +//! fact's value to the new one — an older gateway renewing a certificate would +//! publish a stale chain beside the key it just issued, and one writing an +//! attestation would hang a stale field off a quote about a different public +//! key. That is worse than the field being absent, because absent is a case the +//! newer reader already handles — older gateways exist — while stale is one it +//! will silently trust. +//! +//! It is a parameter rather than a table of key prefixes because the code +//! performing the write is the code that knows which of the two it is doing, +//! and because a parameter cannot be forgotten: a new write does not compile +//! until it says. +//! +//! # Top-level only +//! +//! Only the root map is merged. Below it, nothing distinguishes a struct encoded +//! as a map from a `BTreeMap` field: recursing would resurrect map entries that +//! were deliberately removed. So a value type gains new fields **at its root**; +//! a nested type that has to grow needs its own preservation (a +//! `#[serde(flatten)]` catch-all), and this module declines the merge for such a +//! type rather than half-doing it. +//! +//! A field this binary stops declaring is carried forever, since it is +//! indistinguishable from a field a newer peer wrote. Retiring one for real +//! needs an explicit list of names to drop on write; until something needs to be +//! retired, that list would have no entries to hold. + +use rmp::Marker; +use serde::de::{self, Deserializer, Visitor}; +use std::fmt; +use tracing::debug; + +/// Append the fields of `stored` that `T` does not declare to `encoded`. +/// +/// Returns `encoded` untouched — today's behavior — whenever the merge cannot be +/// made safely: `T` is not a plain struct, either side is not a string-keyed +/// map, either side does not parse cleanly, or the merged bytes do not read +/// back as `T`. Preserving unknown fields is an improvement on the write path, +/// never a precondition for the write. +pub fn carry_unknown_fields<T: serde::de::DeserializeOwned>( + key: &str, + declared: &[&str], + stored: &[u8], + encoded: Vec<u8>, +) -> Vec<u8> { + let Some((_, stored_fields)) = split_map(stored) else { + return encoded; + }; + // The steady state is a single-version cluster, where the stored record + // holds nothing this binary has not heard of. Settle that before parsing + // the outgoing encoding, so the common write pays for one walk, not two. + if stored_fields + .iter() + .all(|field| declared.contains(&field.name)) + { + return encoded; + } + let Some((header_len, encoded_fields)) = split_map(&encoded) else { + return encoded; + }; + + let carried: Vec<&Field> = stored_fields + .iter() + .filter(|field| { + !declared.contains(&field.name) + && !encoded_fields.iter().any(|out| out.name == field.name) + }) + .collect(); + if carried.is_empty() { + return encoded; + } + + let extra: usize = carried.iter().map(|field| field.raw.len()).sum(); + let mut out = Vec::with_capacity(encoded.len() + extra + MAX_MAP_HEADER_LEN); + if write_map_header(&mut out, encoded_fields.len() + carried.len()).is_none() { + return encoded; + } + out.extend_from_slice(&encoded[header_len..]); + for field in &carried { + out.extend_from_slice(field.raw); + } + + // Never write a record this binary cannot read back. The field list says + // which names `T` knows, not what it tolerates next to them: a + // `deny_unknown_fields` type, or a stored record that repeats a key, would + // otherwise turn a healthy write into a record its own writer can no longer + // decode. Dropping a newer peer's field is the lesser failure — it is the + // one this module exists to reduce, not one it may introduce. + if let Err(err) = super::decode::<T>(&out) { + debug!("declining to carry unknown fields for key {key}: {err:#}"); + return encoded; + } + + debug!( + "carried {} unknown field(s) forward for key {key}: {:?}", + carried.len(), + carried.iter().map(|field| field.name).collect::<Vec<_>>(), + ); + out +} + +/// The field names `T` declares, or `None` if `T` does not deserialize from a +/// plain struct. +/// +/// `None` is the answer for every shape the merge must stay away from: a scalar, +/// an internally tagged enum (whose variants have disjoint fields, so merging +/// across a variant change would mix them), and a type using +/// `#[serde(flatten)]`, which reads as a map and already preserves what it does +/// not know. +pub fn declared_fields<T: serde::de::DeserializeOwned>() -> Option<&'static [&'static str]> { + let mut fields = None; + let _ = T::deserialize(FieldProbe { out: &mut fields }); + fields +} + +/// A deserializer that answers nothing. +/// +/// Its only purpose is the field list the derived `Deserialize` impl passes to +/// `deserialize_struct`; every other entry point refuses, which ends the +/// deserialization immediately. +struct FieldProbe<'a> { + out: &'a mut Option<&'static [&'static str]>, +} + +#[derive(Debug)] +struct ProbeDone; + +impl fmt::Display for ProbeDone { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("field probe") + } +} + +impl std::error::Error for ProbeDone {} + +impl de::Error for ProbeDone { + fn custom<T: fmt::Display>(_msg: T) -> Self { + // Deliberately drops the message: the probe always fails, and formatting + // a string for an error nobody reads would allocate on every write. + ProbeDone + } +} + +impl<'de> Deserializer<'de> for FieldProbe<'_> { + type Error = ProbeDone; + + fn deserialize_struct<V: Visitor<'de>>( + self, + _name: &'static str, + fields: &'static [&'static str], + _visitor: V, + ) -> Result<V::Value, Self::Error> { + *self.out = Some(fields); + Err(ProbeDone) + } + + fn deserialize_newtype_struct<V: Visitor<'de>>( + self, + _name: &'static str, + visitor: V, + ) -> Result<V::Value, Self::Error> { + visitor.visit_newtype_struct(self) + } + + fn deserialize_any<V: Visitor<'de>>(self, _visitor: V) -> Result<V::Value, Self::Error> { + Err(ProbeDone) + } + + serde::forward_to_deserialize_any! { + bool i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 char str string bytes byte_buf + option unit unit_struct seq tuple tuple_struct map enum identifier ignored_any + } +} + +/// A `map32` header is the longest msgpack map header. +const MAX_MAP_HEADER_LEN: usize = 5; + +/// One top-level `key -> value` pair, borrowed from the buffer it was parsed +/// from so that carrying it forward copies the original bytes rather than +/// re-encoding a decoded value. +struct Field<'a> { + name: &'a str, + /// Key and value bytes, contiguous and verbatim. + raw: &'a [u8], +} + +/// Split a msgpack map into its header length and its top-level fields. +/// +/// Returns `None` unless the buffer is exactly one map whose keys are all +/// strings. A scalar, an array (the positional struct encoding older releases +/// wrote), an enum encoded as a string, or trailing bytes all decline rather +/// than get reinterpreted. +fn split_map(buf: &[u8]) -> Option<(usize, Vec<Field<'_>>)> { + let (header_len, count) = match Marker::from_u8(*buf.first()?) { + Marker::FixMap(n) => (1, n as usize), + Marker::Map16 => (3, read_uint(buf, 1, 2)?), + Marker::Map32 => (5, read_uint(buf, 1, 4)?), + _ => return None, + }; + let mut pos = header_len; + let mut fields = Vec::with_capacity(count.min(32)); + for _ in 0..count { + let key_start = pos; + let key_end = skip_value(buf, pos)?; + let name = read_str(buf, key_start, key_end)?; + pos = skip_value(buf, key_end)?; + fields.push(Field { + name, + raw: buf.get(key_start..pos)?, + }); + } + // Trailing bytes mean this is not the value we think it is. + (pos == buf.len()).then_some((header_len, fields)) +} + +/// The offset just past the msgpack value starting at `pos`, or `None` if the +/// input is malformed, truncated, or claims more elements than it has bytes for. +/// +/// Iterative rather than recursive: the stored bytes come off the wire, and a +/// deeply nested value must not be able to exhaust the stack. `pending` stays +/// bounded by the remaining byte count because every element costs at least the +/// one byte of its own marker. +fn skip_value(buf: &[u8], mut pos: usize) -> Option<usize> { + let mut pending: usize = 1; + while pending > 0 { + pending -= 1; + let marker = Marker::from_u8(*buf.get(pos)?); + pos = pos.checked_add(1)?; + let (payload, children) = match marker { + Marker::FixPos(_) | Marker::FixNeg(_) | Marker::Null | Marker::True | Marker::False => { + (0, 0) + } + Marker::U8 | Marker::I8 => (1, 0), + Marker::U16 | Marker::I16 => (2, 0), + Marker::U32 | Marker::I32 | Marker::F32 => (4, 0), + Marker::U64 | Marker::I64 | Marker::F64 => (8, 0), + Marker::FixStr(len) => (len as usize, 0), + Marker::Str8 | Marker::Bin8 => (1 + read_uint(buf, pos, 1)?, 0), + Marker::Str16 | Marker::Bin16 => (2 + read_uint(buf, pos, 2)?, 0), + Marker::Str32 | Marker::Bin32 => (4 + read_uint(buf, pos, 4)?, 0), + // One type byte, then the payload. + Marker::FixExt1 => (1 + 1, 0), + Marker::FixExt2 => (1 + 2, 0), + Marker::FixExt4 => (1 + 4, 0), + Marker::FixExt8 => (1 + 8, 0), + Marker::FixExt16 => (1 + 16, 0), + Marker::Ext8 => (1 + 1 + read_uint(buf, pos, 1)?, 0), + Marker::Ext16 => (2 + 1 + read_uint(buf, pos, 2)?, 0), + Marker::Ext32 => (4 + 1 + read_uint(buf, pos, 4)?, 0), + Marker::FixArray(len) => (0, len as usize), + Marker::Array16 => (2, read_uint(buf, pos, 2)?), + Marker::Array32 => (4, read_uint(buf, pos, 4)?), + Marker::FixMap(len) => (0, (len as usize).checked_mul(2)?), + Marker::Map16 => (2, read_uint(buf, pos, 2)?.checked_mul(2)?), + Marker::Map32 => (4, read_uint(buf, pos, 4)?.checked_mul(2)?), + // Never used by the spec, and rejected by rmp-serde on read. + Marker::Reserved => return None, + }; + pos = pos.checked_add(payload)?; + if pos > buf.len() { + return None; + } + pending = pending.checked_add(children)?; + if pending > buf.len() - pos { + return None; + } + } + Some(pos) +} + +/// A big-endian unsigned integer of `size` bytes at `pos`. +fn read_uint(buf: &[u8], pos: usize, size: usize) -> Option<usize> { + let bytes = buf.get(pos..pos.checked_add(size)?)?; + bytes.iter().try_fold(0usize, |acc, byte| { + acc.checked_mul(256)?.checked_add(*byte as usize) + }) +} + +/// The contents of the msgpack string spanning `start..end`. +fn read_str(buf: &[u8], start: usize, end: usize) -> Option<&str> { + let body = match Marker::from_u8(*buf.get(start)?) { + Marker::FixStr(_) => start + 1, + Marker::Str8 => start + 2, + Marker::Str16 => start + 3, + Marker::Str32 => start + 5, + _ => return None, + }; + std::str::from_utf8(buf.get(body..end)?).ok() +} + +fn write_map_header(out: &mut Vec<u8>, count: usize) -> Option<()> { + match count { + 0..=15 => out.push(0x80 | count as u8), + 16..=0xffff => { + out.push(0xde); + out.extend_from_slice(&(count as u16).to_be_bytes()); + } + _ => { + out.push(0xdf); + out.extend_from_slice(&u32::try_from(count).ok()?.to_be_bytes()); + } + } + Some(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::{Rng, SeedableRng}; + use serde::{Deserialize, Serialize}; + use std::collections::BTreeMap; + + /// A record as an older build declares it. + #[derive(Debug, Default, Serialize, Deserialize)] + struct Old { + app_id: String, + ip: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + note: Option<String>, + } + + /// The same record after a later release added fields to it. + #[derive(Debug, Default, Serialize, Deserialize)] + struct New { + app_id: String, + ip: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + note: Option<String>, + #[serde(default)] + admin_port_policy: Option<u32>, + #[serde(default)] + labels: BTreeMap<String, String>, + } + + fn stored_by_a_newer_build() -> Vec<u8> { + rmp_serde::to_vec_named(&New { + app_id: "app".into(), + ip: "10.0.0.1".into(), + note: Some("keep".into()), + admin_port_policy: Some(7), + labels: BTreeMap::from([("tier".to_string(), "edge".to_string())]), + }) + .unwrap() + } + + /// The point of the module: an older gateway rewriting a record must not + /// silently delete what a newer peer put there. + #[test] + fn a_field_only_a_newer_build_knows_survives_an_older_writer() { + let stored = stored_by_a_newer_build(); + + let mut record: Old = rmp_serde::from_slice(&stored).unwrap(); + record.ip = "10.0.0.2".into(); + let encoded = rmp_serde::to_vec_named(&record).unwrap(); + + let written = carry_unknown_fields::<Old>( + "inst/app", + declared_fields::<Old>().unwrap(), + &stored, + encoded, + ); + let seen_by_new: New = rmp_serde::from_slice(&written).unwrap(); + assert_eq!( + seen_by_new.ip, "10.0.0.2", + "the writer's own change applies" + ); + assert_eq!(seen_by_new.admin_port_policy, Some(7)); + assert_eq!(seen_by_new.labels["tier"], "edge"); + // And the result is still a record the writer itself can read back. + assert_eq!( + rmp_serde::from_slice::<Old>(&written).unwrap().ip, + "10.0.0.2" + ); + } + + /// Deciding "unknown" from the keys missing in the new encoding rather than + /// from what the type declares would undo a deliberate clear: `note` is + /// skipped when it is `None`, so it is absent from the encoding for the same + /// reason a field from the future is. + #[test] + fn a_field_this_build_declares_is_not_resurrected() { + let stored = stored_by_a_newer_build(); + + let mut record: Old = rmp_serde::from_slice(&stored).unwrap(); + assert_eq!(record.note.as_deref(), Some("keep")); + record.note = None; + let encoded = rmp_serde::to_vec_named(&record).unwrap(); + + let written = carry_unknown_fields::<Old>( + "inst/app", + declared_fields::<Old>().unwrap(), + &stored, + encoded, + ); + let seen_by_new: New = rmp_serde::from_slice(&written).unwrap(); + assert_eq!(seen_by_new.note, None, "the clear must stick"); + assert_eq!( + seen_by_new.admin_port_policy, + Some(7), + "unknowns still carried" + ); + } + + /// Anything that is not a string-keyed map is left exactly as encoded: a + /// scalar, and the positional array encoding that older releases wrote + /// before values became named maps. + #[test] + fn a_value_that_is_not_a_string_keyed_map_is_left_alone() { + let encoded = rmp_serde::to_vec_named(&Old::default()).unwrap(); + for stored in [ + rmp_serde::to_vec_named(&42u64).unwrap(), + rmp_serde::to_vec_named(&"up").unwrap(), + rmp_serde::to_vec(&New::default()).unwrap(), + b"not msgpack at all".to_vec(), + Vec::new(), + ] { + let written = carry_unknown_fields::<Old>( + "inst/app", + declared_fields::<Old>().unwrap(), + &stored, + encoded.clone(), + ); + assert_eq!( + written, encoded, + "stored {stored:?} must not change the write" + ); + } + + // A scalar value type never gets as far as the merge. + assert!(declared_fields::<u64>().is_none()); + } + + #[derive(Debug, Serialize, Deserialize)] + struct Flattened { + a: u8, + #[serde(flatten)] + rest: BTreeMap<String, u8>, + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + enum Tagged { + Cloudflare { api_token: String }, + Route53 { access_key: String }, + } + + /// The probe must refuse every shape whose fields it cannot enumerate. + /// An internally tagged enum is the dangerous one: its variants have + /// disjoint fields, so merging across a variant change would graft one + /// variant's secrets onto another. + #[test] + fn a_type_the_probe_cannot_enumerate_declines_the_merge() { + assert_eq!( + declared_fields::<Old>(), + Some(&["app_id", "ip", "note"][..]) + ); + assert_eq!(declared_fields::<Flattened>(), None); + assert_eq!(declared_fields::<Tagged>(), None); + assert_eq!(declared_fields::<u64>(), None); + assert_eq!(declared_fields::<Vec<u8>>(), None); + assert_eq!(declared_fields::<Option<Old>>(), None); + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(deny_unknown_fields)] + struct Strict { + app_id: String, + ip: String, + } + + /// The declared field list says which names `T` knows, not what it tolerates + /// beside them. Carrying a field into a `deny_unknown_fields` type would + /// leave a record its own writer can no longer decode — strictly worse than + /// the field being dropped, which is the failure this module already + /// accepts everywhere else. + #[test] + fn a_merge_that_would_not_read_back_is_abandoned() { + let stored = stored_by_a_newer_build(); + let encoded = rmp_serde::to_vec_named(&Strict { + app_id: "app".into(), + ip: "10.0.0.2".into(), + }) + .unwrap(); + + let written = + carry_unknown_fields::<Strict>("inst/app", &["app_id", "ip"], &stored, encoded.clone()); + assert_eq!( + written, encoded, + "the write must stay decodable by its writer" + ); + rmp_serde::from_slice::<Strict>(&written).expect("writer reads its own record"); + } + + /// A `BTreeMap` value and an externally tagged enum both encode as maps, so + /// the byte walker alone cannot tell them apart from a struct: carrying a + /// "missing" key into a map would resurrect an entry that was deliberately + /// removed, and into an enum it would graft one variant's fields onto + /// another. The declared-field probe is what refuses them. + #[test] + fn map_shaped_values_that_are_not_structs_are_refused() { + #[derive(Debug, Serialize, Deserialize)] + enum Tagless { + Cloudflare { api_token: String }, + Route53 { access_key: String }, + } + + // Both are maps on the wire, which is all the byte walker can see. + for bytes in [ + rmp_serde::to_vec_named(&BTreeMap::from([("k".to_string(), 1u8)])).unwrap(), + rmp_serde::to_vec_named(&Tagless::Cloudflare { + api_token: "secret".into(), + }) + .unwrap(), + ] { + assert!( + matches!(bytes[0], 0x80..=0x8f), + "the fixture must be a map on the wire for this test to mean anything" + ); + } + + assert!( + declared_fields::<BTreeMap<String, u8>>().is_none(), + "a removed map entry must stay removed" + ); + assert!( + declared_fields::<Tagless>().is_none(), + "one variant's fields must not follow a change of variant" + ); + } + + #[derive(Debug, Serialize, Deserialize)] + struct Renamed { + #[serde(rename = "app")] + app_id: String, + #[serde(alias = "old_ip")] + ip: String, + } + + /// A renamed field is declared under its wire name, and an alias counts as + /// declared too — otherwise a record written under the alias would be + /// carried forward next to the canonical key and then fail to decode as a + /// duplicate field. + #[test] + fn renames_and_aliases_count_as_declared() { + assert_eq!( + declared_fields::<Renamed>(), + Some(&["app", "ip", "old_ip"][..]) + ); + } + + /// A map with more than 15 entries needs a wider header than the one it + /// started with, so the rewrite has to widen it rather than patch the + /// original byte. + #[test] + fn the_map_header_widens_when_the_field_count_outgrows_it() { + let wide: BTreeMap<String, u8> = (0..15).map(|i| (format!("f{i}"), i as u8)).collect(); + let stored = rmp_serde::to_vec_named(&wide).unwrap(); + assert_eq!(stored[0], 0x8f, "15 fields still fit a fixmap"); + + let encoded = rmp_serde::to_vec_named(&Old::default()).unwrap(); + let written = carry_unknown_fields::<Old>( + "inst/app", + declared_fields::<Old>().unwrap(), + &stored, + encoded, + ); + assert_eq!(written[0], 0xde, "17 fields need a map16 header"); + assert_eq!(split_map(&written).unwrap().1.len(), 15 + 2); + } + + // ---- differential tests against a full msgpack decoder ---- + + fn random_value(rng: &mut impl Rng, depth: u32) -> rmpv::Value { + let leaves = 9; + let arms = if depth == 0 { leaves } else { leaves + 3 }; + match rng.gen_range(0..arms) { + 0 => rmpv::Value::Nil, + 1 => rmpv::Value::Boolean(rng.gen()), + 2 => rmpv::Value::from(rng.gen::<u64>()), + 3 => rmpv::Value::from(rng.gen::<i64>()), + 4 => rmpv::Value::from(rng.gen::<f32>()), + 5 => rmpv::Value::from(rng.gen::<f64>()), + // Long enough to reach the str16/str32 and bin16/bin32 headers. + 6 => rmpv::Value::from("x".repeat(rng.gen_range(0..70_000))), + 7 => rmpv::Value::Binary(vec![7; rng.gen_range(0..70_000)]), + 8 => rmpv::Value::Ext(rng.gen(), vec![9; rng.gen_range(0..20)]), + 9 => rmpv::Value::Array( + (0..rng.gen_range(0..18)) + .map(|_| random_value(rng, depth - 1)) + .collect(), + ), + 10 => rmpv::Value::Map( + (0..rng.gen_range(0..18)) + .map(|_| (random_value(rng, depth - 1), random_value(rng, depth - 1))) + .collect(), + ), + _ => rmpv::Value::from(rng.gen_range(-32i8..0)), + } + } + + /// The skipper walks the format by hand, so it is only trustworthy if it + /// lands on exactly the same byte a real decoder does — for every header + /// width, not just the compact ones a struct usually produces. + #[test] + fn skipping_a_value_lands_where_a_real_decoder_lands() { + let mut rng = rand::rngs::StdRng::seed_from_u64(7); + for case in 0..2_000 { + let value = random_value(&mut rng, 4); + let mut buf = Vec::new(); + rmpv::encode::write_value(&mut buf, &value).unwrap(); + assert_eq!(skip_value(&buf, 0), Some(buf.len()), "case {case}"); + } + } + + /// The stored bytes arrive over the network, so corruption is the expected + /// input, not the exceptional one. Whatever the skipper accepts, a real + /// decoder must accept identically; whatever it rejects must be genuinely + /// undecodable. + #[test] + fn corrupted_encodings_are_judged_the_same_way_a_real_decoder_judges_them() { + let mut rng = rand::rngs::StdRng::seed_from_u64(23); + let mut accepted = 0u32; + for _ in 0..50_000 { + let value = random_value(&mut rng, 3); + let mut buf = Vec::new(); + rmpv::encode::write_value(&mut buf, &value).unwrap(); + if buf.is_empty() || buf.len() > 4096 { + continue; + } + for _ in 0..rng.gen_range(1..4) { + let at = rng.gen_range(0..buf.len()); + buf[at] ^= 1 << rng.gen_range(0..8); + } + if rng.gen_range(0..3) == 0 { + buf.truncate(rng.gen_range(0..buf.len())); + } + + let mine = skip_value(&buf, 0); + let mut cursor = &buf[..]; + let theirs = rmpv::decode::read_value(&mut cursor) + .ok() + .map(|_| buf.len() - cursor.len()); + match (mine, theirs) { + (Some(mine), Some(theirs)) => { + accepted += 1; + assert_eq!(mine, theirs, "length disagreement on {buf:?}"); + } + (Some(mine), None) => panic!("accepted {mine} bytes rmpv rejects: {buf:?}"), + (None, Some(_)) => { + assert!(buf.contains(&0xc1), "rejected what rmpv accepts: {buf:?}") + } + (None, None) => {} + } + } + assert!( + accepted > 1_000, + "corpus degenerate: only {accepted} accepted" + ); + } + + /// Nothing about the stored bytes is guaranteed, so no input may read past + /// the buffer, panic, or claim more than it was given. + #[test] + fn arbitrary_bytes_are_never_over_read() { + let mut rng = rand::rngs::StdRng::seed_from_u64(11); + for _ in 0..100_000 { + let buf: Vec<u8> = (0..rng.gen_range(0..40)).map(|_| rng.gen()).collect(); + if let Some(end) = skip_value(&buf, 0) { + assert!(end <= buf.len()); + } + if let Some((_, fields)) = split_map(&buf) { + for field in fields { + assert!(field.raw.len() <= buf.len()); + } + } + } + } + + /// A nested value deep enough to exhaust a recursive walker must still be + /// handled, since a peer's record is not bounded by what this node writes. + #[test] + fn deep_nesting_does_not_exhaust_the_stack() { + let deep: Vec<u8> = std::iter::repeat_n(0x91, 200_000).chain([0xc0]).collect(); + assert_eq!(skip_value(&deep, 0), Some(deep.len())); + } + + /// A header that claims more entries than the buffer could possibly hold is + /// the cheap way to make a parser allocate or spin. + #[test] + fn a_header_claiming_more_than_the_buffer_holds_is_refused() { + assert_eq!(skip_value(&[0xdf, 0xff, 0xff, 0xff, 0xff], 0), None); + assert_eq!(skip_value(&[0xdd, 0xff, 0xff, 0xff, 0xff], 0), None); + assert_eq!(skip_value(&[0xdb, 0xff, 0xff, 0xff, 0xff], 0), None); + assert_eq!( + skip_value(&[0xc1], 0), + None, + "the reserved marker is not a value" + ); + assert!(split_map(&[0xdf, 0xff, 0xff, 0xff, 0xff]).is_none()); + } +} diff --git a/dstack/gateway/src/kv/https_client.rs b/dstack/gateway/src/kv/https_client.rs index d0d034a9b..a3854cfb8 100644 --- a/dstack/gateway/src/kv/https_client.rs +++ b/dstack/gateway/src/kv/https_client.rs @@ -5,12 +5,12 @@ //! HTTPS client with mTLS and custom certificate verification during TLS handshake. use std::fmt::Debug; -use std::io::{Read, Write}; +use std::io::Write; use std::sync::Arc; use anyhow::{Context, Result}; -use flate2::{read::GzDecoder, write::GzEncoder, Compression}; -use http_body_util::{BodyExt, Full}; +use flate2::{write::GzEncoder, Compression}; +use http_body_util::{BodyExt, Full, Limited}; use hyper::body::Bytes; use hyper_rustls::HttpsConnectorBuilder; use hyper_util::{ @@ -23,7 +23,23 @@ use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, UnixTime}; use rustls::{DigitallySignedStruct, SignatureScheme}; use serde::{de::DeserializeOwned, Serialize}; -use super::{decode, encode}; +/// Read a peer's response body, refusing one larger than the routes accept on a request. +/// +/// `Body::collect` reads to completion, so without this a peer could stream an unbounded +/// response and the decompression limit downstream would never be reached — the memory +/// is already gone by then. +async fn read_body_bounded(body: hyper::body::Incoming) -> Result<Bytes> { + Limited::new(body, super::MAX_COMPRESSED_SYNC_BYTES) + .collect() + .await + .map(|collected| collected.to_bytes()) + .map_err(|err| { + anyhow::anyhow!( + "failed to read response body (limit {} bytes): {err}", + super::MAX_COMPRESSED_SYNC_BYTES + ) + }) +} /// Custom certificate validator trait for TLS handshake verification. /// @@ -37,14 +53,19 @@ pub trait CertValidator: Debug + Send + Sync + 'static { fn validate(&self, cert_der: &[u8]) -> Result<(), String>; } -/// TLS configuration for mTLS with optional custom certificate validation +/// TLS configuration for mTLS with custom certificate validation #[derive(Clone)] pub struct HttpsClientConfig { pub cert_path: String, pub key_path: String, pub ca_cert_path: String, - /// Optional custom certificate validator (checked during TLS handshake) - pub cert_validator: Option<Arc<dyn CertValidator>>, + /// Custom certificate validator, checked against the peer's certificate + /// during the handshake. + /// + /// Not optional: this is the only thing that distinguishes a gateway in our own + /// cluster from any other holder of a certificate the shared CA signed, and CA-path + /// validation alone does not make that distinction. + pub cert_validator: Arc<dyn CertValidator>, } /// Wrapper that adapts a CertValidator to rustls ServerCertVerifier @@ -131,16 +152,55 @@ impl ServerCertVerifier for CustomCertVerifier { type HyperClient = Client<hyper_rustls::HttpsConnector<HttpConnector>, Full<Bytes>>; -/// HTTPS client with mTLS and optional custom certificate validation. +/// HTTPS client with mTLS and peer identity validation. /// -/// When a `cert_validator` is set in `TlsConfig`, the client runs the validator -/// during the TLS handshake, before any application data is sent. +/// The `cert_validator` runs during the TLS handshake, before any application data is +/// sent, on top of the chain verification against the configured CA. #[derive(Clone)] pub struct HttpsClient { client: HyperClient, } +/// A non-success HTTP status, kept as a typed error in the chain so a caller +/// can react to a specific code. The sync path needs to tell an HTTP 403 +/// rejection apart from a peer that is down or broken without string-matching +/// a formatted error message. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HttpStatusError(pub u16); + +impl std::fmt::Display for HttpStatusError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "request failed with HTTP status {}", self.0) + } +} + +impl std::error::Error for HttpStatusError {} + impl HttpsClient { + async fn post_gzipped( + &self, + url: &str, + body: Vec<u8>, + ) -> Result<hyper::Response<hyper::body::Incoming>> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::fast()); + encoder + .write_all(&body) + .context("failed to compress request")?; + let compressed = encoder.finish().context("failed to finish compression")?; + + let request = hyper::Request::builder() + .method(hyper::Method::POST) + .uri(url) + .header("content-type", "application/x-msgpack-gz") + .body(Full::new(Bytes::from(compressed))) + .context("failed to build request")?; + + self.client + .request(request) + .await + .with_context(|| format!("failed to send request to {url}")) + } + /// Create a new HTTPS client with mTLS configuration pub fn new(tls: &HttpsClientConfig) -> Result<Self> { // Load client certificate and key @@ -166,22 +226,17 @@ impl HttpsClient { .next() .context("no CA certificate found")?; - // Build rustls config with custom verifier if validator is provided - let tls_config_builder = rustls::ClientConfig::builder(); - - let tls_config = if let Some(ref validator) = tls.cert_validator { - let verifier = CustomCertVerifier::new(validator.clone(), ca_cert)?; - tls_config_builder - .dangerous() - .with_custom_certificate_verifier(Arc::new(verifier)) - } else { - // Standard verification without custom validator - let mut root_store = rustls::RootCertStore::empty(); - root_store.add(ca_cert).context("failed to add CA cert")?; - tls_config_builder.with_root_certificates(root_store) - } - .with_client_auth_cert(certs, key) - .context("failed to set client auth cert")?; + // `CustomCertVerifier` runs the validator *after* rustls has verified the chain + // against `ca_cert`, so this is the CA path plus an identity check, never a + // replacement for it. There is deliberately no validator-less branch: it would + // accept any certificate the CA signed, which is every gateway in every cluster + // that shares the CA. + let verifier = CustomCertVerifier::new(tls.cert_validator.clone(), ca_cert)?; + let tls_config = rustls::ClientConfig::builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(verifier)) + .with_client_auth_cert(certs, key) + .context("failed to set client auth cert")?; let https = HttpsConnectorBuilder::new() .with_tls_config(tls_config) @@ -215,66 +270,36 @@ impl HttpsClient { .with_context(|| format!("failed to send request to {url}"))?; if !response.status().is_success() { - anyhow::bail!("request failed: {}", response.status()); + return Err(HttpStatusError(response.status().as_u16()).into()); } - let body = response - .into_body() - .collect() - .await - .context("failed to read response body")? - .to_bytes(); + // Bounded like every other response: this is the bootnode GetPeers path, and + // the threat model does not assume a bootnode is honest. + let body = read_body_bounded(response.into_body()).await?; serde_json::from_slice(&body).context("failed to parse response") } - /// Send a POST request with msgpack + gzip encoded body and receive msgpack + gzip response - pub async fn post_compressed_msg<T: Serialize, R: DeserializeOwned>( - &self, - url: &str, - body: &T, - ) -> Result<R> { - let encoded = encode(body).context("failed to encode request body")?; + /// Send an already-encoded body and return the decompressed response bytes. + pub async fn post_bytes_response(&self, url: &str, body: Vec<u8>) -> Result<Vec<u8>> { + let response = self.post_gzipped(url, body).await?; - // Compress with gzip - let mut encoder = GzEncoder::new(Vec::new(), Compression::fast()); - encoder - .write_all(&encoded) - .context("failed to compress request")?; - let compressed = encoder.finish().context("failed to finish compression")?; - - let request = hyper::Request::builder() - .method(hyper::Method::POST) - .uri(url) - .header("content-type", "application/x-msgpack-gz") - .body(Full::new(Bytes::from(compressed))) - .context("failed to build request")?; + let status = response.status(); + if !status.is_success() { + return Err(HttpStatusError(status.as_u16()).into()); + } - let response = self - .client - .request(request) - .await - .with_context(|| format!("failed to send request to {url}"))?; + let body = read_body_bounded(response.into_body()).await?; + crate::kv::gunzip_bounded(&body, crate::kv::MAX_DECOMPRESSED_SYNC_BYTES) + } + /// Send an already-encoded body to an endpoint whose successful response has no body. + pub async fn post_bytes_no_response(&self, url: &str, body: Vec<u8>) -> Result<()> { + let response = self.post_gzipped(url, body).await?; if !response.status().is_success() { - anyhow::bail!("request failed: {}", response.status()); + return Err(HttpStatusError(response.status().as_u16()).into()); } - - let body = response - .into_body() - .collect() - .await - .context("failed to read response body")? - .to_bytes(); - - // Decompress - let mut decoder = GzDecoder::new(body.as_ref()); - let mut decompressed = Vec::new(); - decoder - .read_to_end(&mut decompressed) - .context("failed to decompress response")?; - - decode(&decompressed).context("failed to decode response") + Ok(()) } } @@ -320,3 +345,332 @@ impl CertValidator for AppIdValidator { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use ra_tls::cert::CertRequest; + use ra_tls::rcgen::KeyPair; + + /// A certificate carrying `PHALA_RATLS_APP_ID`, minted in process. + /// + /// No TEE is involved: `CertRequest` writes the extension unconditionally, and the + /// validator below never looks at a quote — it parses DER and compares bytes. + fn cert_with_app_id(app_id: &[u8]) -> Vec<u8> { + let key = KeyPair::generate().expect("key"); + CertRequest::builder() + .key(&key) + .subject("peer.test") + .app_id(app_id) + .build() + .self_signed() + .expect("self-signed cert") + .der() + .to_vec() + } + + fn cert_without_app_id() -> Vec<u8> { + let key = KeyPair::generate().expect("key"); + CertRequest::builder() + .key(&key) + .subject("peer.test") + .build() + .self_signed() + .expect("self-signed cert") + .der() + .to_vec() + } + + /// The client half of the same rule the sync routes enforce on inbound requests. + /// + /// This runs during the TLS handshake, so a validator that always returns `Ok(())` + /// means this gateway will complete a mutually-authenticated connection to any peer + /// presenting any certificate our CA signed — and then send it our state. Replacing + /// the whole body with `Ok(())`, or inverting the comparison, left the suite green. + #[test] + fn a_peer_certificate_is_accepted_only_when_its_app_id_matches() { + let ours = b"app-id-of-this-cluster".to_vec(); + let validator = AppIdValidator::new(ours.clone()); + + assert_eq!(validator.validate(&cert_with_app_id(&ours)), Ok(())); + assert!( + validator + .validate(&cert_with_app_id(b"a-different-app")) + .is_err(), + "a certificate from another app must not complete the handshake" + ); + } + + /// A certificate that says nothing about which app holds it proves nothing, and must + /// be refused rather than treated as unconstrained. + #[test] + fn a_peer_certificate_without_an_app_id_is_refused() { + let validator = AppIdValidator::new(b"app-id-of-this-cluster".to_vec()); + let err = validator + .validate(&cert_without_app_id()) + .expect_err("a certificate with no app identity must be refused"); + assert!(err.contains("app_id"), "{err}"); + } + + /// Anything that is not a certificate is a parse failure, not a pass. + #[test] + fn a_malformed_certificate_is_refused() { + let validator = AppIdValidator::new(b"whatever".to_vec()); + assert!(validator.validate(b"not a certificate at all").is_err()); + } +} + +/// Response handling tested against a real TLS peer. +/// +/// No container and no TEE: a local listener with a certificate minted in process. +#[cfg(test)] +mod transport_tests { + use super::*; + use hyper::service::service_fn; + use hyper::{Response, StatusCode}; + use hyper_util::rt::TokioIo; + use std::convert::Infallible; + use tokio::net::TcpListener; + use tokio_rustls::TlsAcceptor; + + /// The app id the HTTP-level tests below run under. They are not about identity, + /// but the validator is not optional, so their server and client agree on one. + const TEST_APP_ID: &[u8] = b"app-id-of-this-cluster"; + + /// A CA plus a leaf valid for 127.0.0.1, written where `HttpsClient::new` expects. + fn tls_material(dir: &std::path::Path) -> (HttpsClientConfig, Vec<u8>, Vec<u8>) { + app_id_server_cert(dir, TEST_APP_ID) + } + + /// Serve one fixed response over TLS and return the URL to reach it. + async fn serve(status: StatusCode, body: Vec<u8>, cert: Vec<u8>, key: Vec<u8>) -> String { + let certs = vec![rustls::pki_types::CertificateDer::from(cert)]; + let key = rustls::pki_types::PrivateKeyDer::try_from(key).expect("server key"); + let config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certs, key) + .expect("server config"); + let acceptor = TlsAcceptor::from(Arc::new(config)); + + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("addr"); + + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + let acceptor = acceptor.clone(); + let body = body.clone(); + tokio::spawn(async move { + let Ok(tls) = acceptor.accept(stream).await else { + return; + }; + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection( + TokioIo::new(tls), + service_fn(move |_req| { + let body = body.clone(); + async move { + Ok::<_, Infallible>( + Response::builder() + .status(status) + .body(Full::new(Bytes::from(body))) + .expect("response"), + ) + } + }), + ) + .await; + }); + } + }); + + format!("https://127.0.0.1:{}/wavekv/sync/persistent", addr.port()) + } + + fn gzip(bytes: &[u8]) -> Vec<u8> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::fast()); + encoder.write_all(bytes).expect("gzip"); + encoder.finish().expect("gzip finish") + } + + async fn request(status: StatusCode, body: Vec<u8>) -> Result<Vec<u8>> { + let _ = rustls::crypto::ring::default_provider().install_default(); + let dir = tempfile::tempdir().expect("tempdir"); + let (config, cert, key) = tls_material(dir.path()); + let url = serve(status, body, cert, key).await; + HttpsClient::new(&config) + .expect("client") + .post_bytes_response(&url, b"request".to_vec()) + .await + } + + /// The status code survives into the error chain as a typed value: the + /// sync path tells an HTTP 403 rejection apart from a peer that is down or + /// broken, and string-matching a Display line is not a contract. + #[tokio::test] + async fn a_refusal_status_is_readable_from_the_error_chain() { + let err = request(StatusCode::FORBIDDEN, Vec::new()) + .await + .expect_err("403 must not read as success"); + let status = err + .chain() + .filter_map(|cause| cause.downcast_ref::<HttpStatusError>()) + .next() + .expect("the status must be present as a typed error"); + assert_eq!(*status, HttpStatusError(403)); + } + + /// A non-success status must never be decoded as a successful sync response. + #[tokio::test] + async fn a_server_error_is_rejected() { + assert!(request(StatusCode::INTERNAL_SERVER_ERROR, Vec::new()) + .await + .is_err()); + assert!(request(StatusCode::BAD_REQUEST, Vec::new()).await.is_err()); + } + + /// A peer that answers gets its body decompressed and returned. + #[tokio::test] + async fn an_upgraded_peer_returns_its_decoded_body() { + let payload = b"the-envelope-bytes".to_vec(); + let got = request(StatusCode::OK, gzip(&payload)).await.unwrap(); + assert_eq!(got, payload); + } + + /// Push responses intentionally have no body. A successful delivery must not be + /// passed through the sync-response gunzip path. + #[tokio::test] + async fn an_empty_success_response_is_accepted_for_a_push() { + let _ = rustls::crypto::ring::default_provider().install_default(); + let dir = tempfile::tempdir().expect("tempdir"); + let (config, cert, key) = tls_material(dir.path()); + let url = serve(StatusCode::OK, Vec::new(), cert, key).await; + + HttpsClient::new(&config) + .expect("client") + .post_bytes_no_response(&url, b"push-envelope".to_vec()) + .await + .expect("an empty 200 response is a successful push"); + } + + #[tokio::test] + async fn a_failed_push_status_is_rejected() { + let _ = rustls::crypto::ring::default_provider().install_default(); + let dir = tempfile::tempdir().expect("tempdir"); + let (config, cert, key) = tls_material(dir.path()); + let url = serve(StatusCode::NOT_FOUND, Vec::new(), cert, key).await; + + assert!(HttpsClient::new(&config) + .expect("client") + .post_bytes_no_response(&url, b"push-envelope".to_vec()) + .await + .is_err()); + } + + /// A server certificate carrying an app id, signed by the same test CA. + /// + /// The client config that comes back validates against that same app id, which is + /// what the HTTP-level tests want. `a_peer_from_another_app_cannot_complete_the_handshake` + /// overrides `cert_validator` to make the two disagree. + fn app_id_server_cert( + dir: &std::path::Path, + app_id: &[u8], + ) -> (HttpsClientConfig, Vec<u8>, Vec<u8>) { + let pki = ra_tls::test_pki::write_mtls_pki( + dir, + ra_tls::test_pki::TestCert::new("peer.test") + .alt_name("127.0.0.1") + .app_id(app_id) + .server_auth(true), + ) + .expect("write test PKI"); + + ( + HttpsClientConfig { + cert_path: pki.cert_path.to_string_lossy().into_owned(), + key_path: pki.key_path.to_string_lossy().into_owned(), + ca_cert_path: pki.ca_cert_path.to_string_lossy().into_owned(), + cert_validator: Arc::new(AppIdValidator::new(app_id.to_vec())), + }, + pki.leaf.cert_der(), + pki.leaf.key_der(), + ) + } + + /// The client-side identity check, over a real handshake rather than a direct call. + /// + /// `AppIdValidator` runs inside `CustomCertVerifier`, which rustls only reaches once + /// standard chain verification passes — so unit-testing the validator alone leaves + /// the wiring untested. A peer from another app must fail to connect at all, before + /// any application bytes move. + #[tokio::test] + async fn a_peer_from_another_app_cannot_complete_the_handshake() { + let _ = rustls::crypto::ring::default_provider().install_default(); + let ours = b"app-id-of-this-cluster".to_vec(); + + for (server_app_id, expect_ok) in + [(ours.clone(), true), (b"a-different-app".to_vec(), false)] + { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut config, cert, key) = app_id_server_cert(dir.path(), &server_app_id); + config.cert_validator = Arc::new(AppIdValidator::new(ours.clone())); + let url = serve(StatusCode::OK, gzip(b"response"), cert, key).await; + + let got = HttpsClient::new(&config) + .expect("client") + .post_bytes_response(&url, b"x".to_vec()) + .await; + + if expect_ok { + assert_eq!( + got.expect("a peer from our own app must connect"), + b"response" + ); + } else { + assert!( + got.is_err(), + "a peer from another app completed the handshake" + ); + } + } + } + + /// `post_json` is the bootnode GetPeers path, and the threat model does not assume a + /// bootnode is honest — so a failure status must not be parsed as a peer list. + #[tokio::test] + async fn a_failed_bootnode_fetch_is_not_parsed_as_peers() { + let _ = rustls::crypto::ring::default_provider().install_default(); + let dir = tempfile::tempdir().expect("tempdir"); + let (config, cert, key) = tls_material(dir.path()); + let url = serve(StatusCode::FORBIDDEN, b"null".to_vec(), cert, key).await; + let client = HttpsClient::new(&config).expect("client"); + let out: Result<Option<u32>> = client.post_json(&url, &()).await; + assert!( + out.is_err(), + "a 403 from a bootnode must not parse as a body" + ); + } + + /// The response body is bounded before it is decompressed, so a peer cannot spend + /// our memory ahead of any decoding limit. + /// + /// The body must be *valid* gzip that merely exceeds the compressed ceiling. A + /// malformed one is rejected by `gunzip_bounded` whatever the ceiling says, so it + /// would pass this test with the bound removed entirely — which is exactly what the + /// first version of it did. Stored-mode gzip keeps the encoded size at roughly the + /// input size, so the payload clears the ceiling while decompressing well inside it. + #[tokio::test] + async fn an_oversized_response_body_is_refused() { + let stored = { + let mut encoder = GzEncoder::new(Vec::new(), Compression::none()); + encoder + .write_all(&vec![0u8; super::super::MAX_COMPRESSED_SYNC_BYTES + 1]) + .expect("gzip"); + encoder.finish().expect("gzip finish") + }; + assert!( + stored.len() > super::super::MAX_COMPRESSED_SYNC_BYTES, + "the fixture depends on the compressed body clearing the ceiling" + ); + assert!(request(StatusCode::OK, stored).await.is_err()); + } +} diff --git a/dstack/gateway/src/kv/import.rs b/dstack/gateway/src/kv/import.rs new file mode 100644 index 000000000..593715379 --- /dev/null +++ b/dstack/gateway/src/kv/import.rs @@ -0,0 +1,522 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Validation boundary between replicated KV records and the local data plane. +//! +//! Every `inst/` record that reaches `ProxyState` — and through it the rendered +//! `wg.conf` — passes through [`accept_instances`] first. The registration-path +//! checks are re-run here because: +//! +//! - a record can arrive from a peer without ever passing through this node's +//! registration RPC, so its checks are not a boundary for synced data; +//! - last-writer-wins replication cannot enforce invariants that span keys, so +//! IP and public-key uniqueness have to be re-established on import; +//! - `wg syncconf` rejects the *whole* config file when a single peer key is +//! malformed, which turns one bad record into a node-wide WireGuard freeze. +//! +//! Validation never aborts the batch: an offending record is skipped and +//! reported, every other instance keeps its routing. +//! +//! Refusals are not all the same, so [`Rejection`] tells the caller which kind +//! it is holding. A record this node cannot make sense of says nothing about +//! whether its instance still exists, so the instance keeps the state the data +//! plane already has; a record that lost an IP or key conflict says the address +//! belongs to someone else, so its instance has to stop being routable. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::net::Ipv4Addr; + +use anyhow::{ensure, Result}; +use base64::{engine::general_purpose::STANDARD, Engine}; + +use super::{InstanceRecord, LoadedInstances, MAX_CLOCK_DRIFT_SECS}; +use crate::config::WgConfig; +use crate::time::now_secs; + +/// A WireGuard public key is 32 raw bytes, base64-encoded by `wg`. +const WG_PUBLIC_KEY_BYTES: usize = 32; + +/// Padded base64 of 32 bytes is always 44 characters. +const WG_PUBLIC_KEY_B64_LEN: usize = 44; + +/// Upper bound on identifier fields carried in a KV record. Real values are +/// hex-encoded hashes (40-64 chars); the bound keeps a corrupt record that +/// still decodes from pushing an arbitrarily long string into the logs and the +/// rendered config. +const MAX_ID_LEN: usize = 128; + +/// Whether a refused record should also cost the instance the state the data +/// plane already holds for it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Rejection { + /// The record is unusable: it fails validation, or its bytes no longer + /// decode. Either way this node cannot tell what the instance looks like + /// now, and the last known-good state is a better answer than none — so + /// whatever the data plane already knows about the instance stays. + Unusable, + /// The record is well-formed but lost a uniqueness conflict to an older + /// registration. Here the winner genuinely owns the IP or the key, so the + /// loser has to stop being routable; keeping it would put the same address + /// in `wg.conf` twice and hand it traffic that belongs to the winner. + LostConflict, +} + +/// A record that failed validation, along with the reason. +pub struct RejectedInstance { + pub instance_id: String, + pub reason: anyhow::Error, + pub rejection: Rejection, +} + +/// Records accepted for import, plus the ones that were skipped. +pub struct AcceptedInstances { + pub instances: BTreeMap<String, InstanceRecord>, + pub rejected: Vec<RejectedInstance>, +} + +impl AcceptedInstances { + /// Instance IDs that are absent from `instances` only because their record + /// was unreadable, and whose existing data-plane state must therefore be + /// left in place rather than treated as a remote deletion. + pub fn unreadable(&self) -> HashSet<&str> { + self.rejected + .iter() + .filter(|rejected| rejected.rejection == Rejection::Unusable) + .map(|rejected| rejected.instance_id.as_str()) + .collect() + } +} + +/// Validate a WireGuard public key as `wg` itself would accept it. +pub fn validate_wg_public_key(public_key: &str) -> Result<()> { + ensure!(!public_key.is_empty(), "public key is empty"); + ensure!( + !public_key.contains(|c: char| c.is_whitespace() || c.is_control()), + "public key contains whitespace or control characters" + ); + // `wg` writes the padded form and accepts nothing else, so the length is + // part of the format rather than a consequence of the decode below. + ensure!( + public_key.len() == WG_PUBLIC_KEY_B64_LEN, + "public key is {} characters, expected {WG_PUBLIC_KEY_B64_LEN}", + public_key.len() + ); + let decoded = STANDARD + .decode(public_key) + .map_err(|err| anyhow::anyhow!("public key is not valid base64: {err}"))?; + ensure!( + decoded.len() == WG_PUBLIC_KEY_BYTES, + "public key decodes to {} bytes, expected {WG_PUBLIC_KEY_BYTES}", + decoded.len() + ); + Ok(()) +} + +/// Validate an identifier field: non-empty, bounded, and free of whitespace +/// and control characters. Every identifier a legitimate gateway writes into +/// a KV record satisfies this, so it is also the acceptance bound for +/// operator-supplied instance IDs (e.g. `Admin.RemoveCvm`). +pub(crate) fn validate_id(field: &str, value: &str) -> Result<()> { + ensure!(!value.is_empty(), "{field} is empty"); + ensure!( + value.len() <= MAX_ID_LEN, + "{field} is {} bytes, limit is {MAX_ID_LEN}", + value.len() + ); + ensure!( + !value.contains(|c: char| c.is_whitespace() || c.is_control()), + "{field} contains whitespace or control characters" + ); + Ok(()) +} + +/// Per-record checks that do not depend on any other record. +/// +/// `now` is the local wall clock in seconds, taken once per batch so every +/// record in a batch is judged against the same instant. +fn validate_instance( + wg: &WgConfig, + now: u64, + instance_id: &str, + data: &InstanceRecord, +) -> Result<()> { + validate_id("instance_id", instance_id)?; + validate_id("app_id", &data.app_id)?; + validate_wg_public_key(&data.public_key)?; + ensure!( + data.public_key != wg.public_key, + "public key belongs to this gateway" + ); + // The routable network, not this node's allocation share: in a cluster + // every node carries peers for the CVMs registered on the other nodes, and + // those hold addresses from the other nodes' shares by design. + ensure!( + wg.is_routable_client_ip(data.ip), + "ip {} is outside the WireGuard network", + data.ip + ); + // `reg_time` is the registering node's clock at that one instant, so a + // clock only has to be wrong once — during the window before chrony + // converges, or across a time jump — for the future timestamp to be written + // into the KV permanently. It does not heal when the clock does: both the + // "gone from KV" pass and `recycle()` age instances with + // `elapsed().unwrap_or_default()`, which reads a future timestamp as zero + // age, so the instance is immune to remote deletion and to local recycling + // until the process restarts. Same horizon as the handshake observations, + // which are ignored for the same reason. + let horizon = now.saturating_add(MAX_CLOCK_DRIFT_SECS); + ensure!( + data.reg_time <= horizon, + "reg_time {} is more than {MAX_CLOCK_DRIFT_SECS}s ahead of local time ({now})", + data.reg_time + ); + Ok(()) +} + +/// Filter KV instance records down to the ones safe to apply locally. +/// +/// Conflicts on IP or public key are resolved by registration time (oldest +/// wins, ties broken by instance ID) so that every node reaches the same +/// decision from the same KV contents — the local registration path resolves +/// them the same way, by refusing the newcomer. +pub fn accept_instances(wg: &WgConfig, loaded: LoadedInstances) -> AcceptedInstances { + accept_instances_at(wg, loaded, now_secs()) +} + +fn accept_instances_at(wg: &WgConfig, loaded: LoadedInstances, now: u64) -> AcceptedInstances { + let LoadedInstances { + decoded, + undecodable, + } = loaded; + + let mut ordered: Vec<(String, InstanceRecord)> = decoded.into_iter().collect(); + ordered.sort_by(|(left_id, left), (right_id, right)| { + left.reg_time + .cmp(&right.reg_time) + .then_with(|| left_id.cmp(right_id)) + }); + + let mut instances = BTreeMap::new(); + let mut rejected: Vec<RejectedInstance> = undecodable + .into_iter() + .map(|(instance_id, reason)| RejectedInstance { + instance_id, + reason: anyhow::anyhow!(reason), + rejection: Rejection::Unusable, + }) + .collect(); + let mut claimed_ips: HashMap<Ipv4Addr, String> = HashMap::new(); + let mut claimed_keys: HashSet<String> = HashSet::new(); + + for (instance_id, data) in ordered { + let checked = validate_instance(wg, now, &instance_id, &data) + .map_err(|reason| (Rejection::Unusable, reason)) + .and_then(|()| { + if let Some(owner) = claimed_ips.get(&data.ip) { + return Err(( + Rejection::LostConflict, + anyhow::anyhow!("ip {} is already assigned to instance {owner}", data.ip), + )); + } + if claimed_keys.contains(&data.public_key) { + return Err(( + Rejection::LostConflict, + anyhow::anyhow!("public key is already registered to another instance"), + )); + } + Ok(()) + }); + if let Err((rejection, reason)) = checked { + rejected.push(RejectedInstance { + instance_id, + reason, + rejection, + }); + continue; + } + claimed_ips.insert(data.ip, instance_id.clone()); + claimed_keys.insert(data.public_key.clone()); + instances.insert(instance_id, data); + } + + AcceptedInstances { + instances, + rejected, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ipnet::Ipv4Net; + + /// Wall clock the tests validate against; every fixture `reg_time` below is + /// well under it unless the test is about the future-timestamp horizon. + const NOW: u64 = 1_700_000_000; + + fn wg_config() -> WgConfig { + WgConfig { + public_key: key(7), + private_key: "gateway".to_string(), + listen_port: 51820, + ip: "10.0.0.1/24".parse::<Ipv4Net>().unwrap(), + reserved_net: vec!["10.0.0.0/28".parse::<Ipv4Net>().unwrap()], + client_ip_range: "10.0.0.0/24".parse::<Ipv4Net>().unwrap(), + interface: "wg0".to_string(), + config_path: "/tmp/wg.conf".to_string(), + endpoint: "127.0.0.1:51820".to_string(), + } + } + + fn key(seed: u8) -> String { + STANDARD.encode([seed; WG_PUBLIC_KEY_BYTES]) + } + + fn instance(ip: &str, public_key: &str, reg_time: u64) -> InstanceRecord { + InstanceRecord { + app_id: "0123456789abcdef".to_string(), + ip: ip.parse().unwrap(), + public_key: public_key.to_string(), + reg_time, + port_policy: None, + port_policy_hash: String::new(), + health_check: None, + } + } + + fn loaded(records: Vec<(&str, InstanceRecord)>) -> LoadedInstances { + LoadedInstances { + decoded: records + .into_iter() + .map(|(id, data)| (id.to_string(), data)) + .collect(), + undecodable: BTreeMap::new(), + } + } + + fn accept(records: Vec<(&str, InstanceRecord)>) -> AcceptedInstances { + accept_instances_at(&wg_config(), loaded(records), NOW) + } + + #[test] + fn accepts_well_formed_records() { + let accepted = accept(vec![ + ("a", instance("10.0.0.20", &key(1), 100)), + ("b", instance("10.0.0.21", &key(2), 200)), + ]); + assert!(accepted.rejected.is_empty()); + assert_eq!(accepted.instances.len(), 2); + } + + #[test] + fn rejects_keys_wg_would_refuse() { + for bad in [ + "", + "not base64!", + &STANDARD.encode([1u8; 16]), + &format!("{}\nEndpoint = 10.0.0.9:1234", key(3)), + ] { + assert!( + validate_wg_public_key(bad).is_err(), + "accepted public key {bad:?}" + ); + } + assert!(validate_wg_public_key(&key(3)).is_ok()); + } + + #[test] + fn rejects_ids_unfit_for_kv_keys_and_logs() { + let too_long = "a".repeat(MAX_ID_LEN + 1); + for bad in ["", " id", "id ", "in id", "in\nid", too_long.as_str()] { + assert!( + validate_id("instance_id", bad).is_err(), + "accepted id {bad:?}" + ); + } + validate_id("instance_id", "peer-instance").unwrap(); + validate_id("instance_id", &"a".repeat(MAX_ID_LEN)).unwrap(); + } + + #[test] + fn one_bad_record_does_not_drop_the_others() { + let accepted = accept(vec![ + ("bad-key", instance("10.0.0.20", "not-a-key", 100)), + ("good", instance("10.0.0.21", &key(2), 200)), + ]); + assert_eq!(accepted.rejected.len(), 1); + assert_eq!(accepted.rejected[0].instance_id, "bad-key"); + assert!(accepted.instances.contains_key("good")); + } + + #[test] + fn rejects_addresses_belonging_to_this_gateway() { + // The gateway's own wg address, an address in its reserved net, and + // non-unicast garbage must never reach the peer list. + for ip in ["10.0.0.1", "10.0.0.5", "127.0.0.1", "224.0.0.1", "0.0.0.0"] { + let accepted = accept(vec![("x", instance(ip, &key(1), 100))]); + assert!(accepted.instances.is_empty(), "accepted ip {ip}"); + } + } + + #[test] + fn rejects_the_gateways_own_public_key() { + let wg = wg_config(); + let accepted = accept_instances_at( + &wg, + loaded(vec![( + "self-peer", + instance("10.0.0.20", &wg.public_key, 100), + )]), + NOW, + ); + assert!(accepted.instances.is_empty()); + assert_eq!(accepted.rejected.len(), 1); + } + + #[test] + fn duplicate_ip_and_key_claims_resolve_to_the_older_registration() { + let accepted = accept(vec![ + ("new", instance("10.0.0.20", &key(9), 300)), + ("old", instance("10.0.0.20", &key(1), 100)), + ]); + assert_eq!(accepted.instances.len(), 1); + assert!(accepted.instances.contains_key("old")); + + let accepted = accept(vec![ + ("new", instance("10.0.0.21", &key(1), 300)), + ("old", instance("10.0.0.20", &key(1), 100)), + ]); + assert_eq!(accepted.instances.len(), 1); + assert!(accepted.instances.contains_key("old")); + } + + #[test] + fn conflict_resolution_does_not_depend_on_iteration_order() { + let forward = accept(vec![ + ("a", instance("10.0.0.20", &key(1), 100)), + ("b", instance("10.0.0.20", &key(2), 100)), + ]); + let reverse = accept(vec![ + ("b", instance("10.0.0.20", &key(2), 100)), + ("a", instance("10.0.0.20", &key(1), 100)), + ]); + assert_eq!( + forward.instances.keys().collect::<Vec<_>>(), + reverse.instances.keys().collect::<Vec<_>>() + ); + assert!(forward.instances.contains_key("a")); + } + + #[test] + fn a_cluster_peers_address_is_not_judged_by_this_nodes_pool() { + // Every deployment gives each node its own slice, and a CVM is handed + // every gateway as a WireGuard server — so each node must carry peers + // holding addresses out of the other nodes' slices. The two shapes in + // tree disagree on whether those slices sit inside a node's own + // interface network, so neither the pool nor the interface network can + // decide this. + let shapes = [ + // deploy-to-vmm.sh: /18 pools inside a shared /16 interface. + ("10.8.0.1/16", "10.8.0.0/18", "10.8.0.5", "10.8.64.5"), + // The test-run suites: a /24 per node, and no node's interface + // covers another's. + ("10.0.41.1/24", "10.0.41.0/24", "10.0.41.5", "10.0.42.5"), + ]; + for (ip, pool, mine, peers) in shapes { + let gateway_addr = ip.split('/').next().unwrap_or_default(); + let wg = WgConfig { + ip: ip.parse::<Ipv4Net>().unwrap(), + reserved_net: vec![format!("{gateway_addr}/32").parse::<Ipv4Net>().unwrap()], + client_ip_range: pool.parse::<Ipv4Net>().unwrap(), + ..wg_config() + }; + let accepted = accept_instances( + &wg, + loaded(vec![ + ("mine", instance(mine, &key(1), 100)), + ("peers", instance(peers, &key(2), 100)), + // Still refused: this gateway's own address. + ("steals-gateway-ip", instance(gateway_addr, &key(3), 100)), + ]), + ); + assert!(accepted.instances.contains_key("mine"), "{ip}"); + assert!( + accepted.instances.contains_key("peers"), + "{ip}: a peer node's instance was refused, which would leave every \ + gateway serving only its own CVMs" + ); + assert!( + !accepted.instances.contains_key("steals-gateway-ip"), + "{ip}" + ); + } + } + + #[test] + fn rejects_registrations_dated_into_the_future() { + // A future reg_time reads as zero age in every `elapsed()` check, which + // makes the instance immune to both remote deletion and local recycling. + let accepted = accept(vec![ + ( + "future", + instance("10.0.0.20", &key(1), NOW + MAX_CLOCK_DRIFT_SECS + 1), + ), + ( + "skewed", + instance("10.0.0.21", &key(2), NOW + MAX_CLOCK_DRIFT_SECS), + ), + ]); + assert!(!accepted.instances.contains_key("future")); + // Drift inside the horizon is ordinary skew between nodes. + assert!(accepted.instances.contains_key("skewed")); + } + + #[test] + fn a_conflict_loser_is_dropped_but_an_unusable_record_keeps_its_instance() { + // The two rejection kinds drive opposite decisions in the reload pass: + // a conflict loser must lose its routing to the winner, while an + // instance whose record we cannot read keeps what the data plane holds. + let accepted = accept(vec![ + ("loser", instance("10.0.0.20", &key(9), 300)), + ("winner", instance("10.0.0.20", &key(1), 100)), + ("malformed", instance("10.0.0.30", "not-a-key", 100)), + ]); + let kind = |id: &str| { + accepted + .rejected + .iter() + .find(|rejected| rejected.instance_id == id) + .map(|rejected| rejected.rejection) + }; + assert_eq!(kind("loser"), Some(Rejection::LostConflict)); + assert_eq!(kind("malformed"), Some(Rejection::Unusable)); + assert_eq!(accepted.unreadable(), HashSet::from(["malformed"])); + } + + #[test] + fn an_undecodable_record_is_reported_as_unreadable_not_as_absent() { + let accepted = accept_instances_at( + &wg_config(), + LoadedInstances { + decoded: [("good".to_string(), instance("10.0.0.20", &key(1), 100))] + .into_iter() + .collect(), + undecodable: [("corrupt".to_string(), "does not decode".to_string())] + .into_iter() + .collect(), + }, + NOW, + ); + assert!(accepted.instances.contains_key("good")); + assert_eq!(accepted.unreadable(), HashSet::from(["corrupt"])); + } + + #[test] + fn rejects_keys_of_the_wrong_length_before_decoding_them() { + let oversized = "A".repeat(4096); + assert!(validate_wg_public_key(&oversized).is_err()); + // 32 bytes unpadded is 43 characters; `wg` writes the padded form. + assert!(validate_wg_public_key(key(1).trim_end_matches('=')).is_err()); + } +} diff --git a/dstack/gateway/src/kv/mod.rs b/dstack/gateway/src/kv/mod.rs index 07dfcf228..7896d029a 100644 --- a/dstack/gateway/src/kv/mod.rs +++ b/dstack/gateway/src/kv/mod.rs @@ -10,7 +10,7 @@ //! Key schema: //! //! # Persistent WaveKV (needs persistence + sync) -//! - `inst/{instance_id}` → InstanceData +//! - `inst/{instance_id}` → InstanceRecord //! - `node/{node_id}` → NodeData //! - `dns_cred/{cred_id}` → DnsCredential //! - `dns_cred_default` → cred_id (default credential ID) @@ -27,17 +27,40 @@ //! - `conn/{instance_id}/{node_id}` → u64 (connection count) //! - `last_seen/inst/{instance_id}` → u64 (timestamp) //! - `last_seen/node/{node_id}/{seen_by_node_id}` → u64 (timestamp) - +//! +//! The key list above is documentation, not an enforced allowlist, and deliberately so. +//! A gateway terminates TLS, so a peer with code execution already holds the mesh +//! WireGuard keys, the base-domain certificate and every proxied request in plaintext; +//! restricting which keys it may replicate protects nothing that is still standing. It +//! also would not bound the store, since key shape says nothing about volume -- that is +//! what wavekv `Limits` (entry size, clock drift, capacity) is for. What such a check +//! does reach is our own rolling upgrades: a refused entry parks ack adoption for the +//! whole node pair, so a node writing a key its peer does not know silently stops the +//! two from converging. Bound what a peer can consume; do not police what it means. + +mod compat; mod https_client; +pub mod import; mod sync_service; +#[cfg(test)] +pub(crate) use https_client::HttpsClient; pub use https_client::{AppIdValidator, HttpsClientConfig}; -pub use sync_service::{fetch_peers_from_bootnode, WaveKvSyncService}; -use tracing::warn; +pub use sync_service::{fetch_peers_from_bootnode, PersistentWriteNotifier, WaveKvSyncService}; +use tracing::{error, warn}; -use std::{collections::BTreeMap, net::Ipv4Addr, path::Path, time::Duration}; +use std::{ + collections::{BTreeMap, BTreeSet}, + net::Ipv4Addr, + path::Path, + time::Duration, +}; use anyhow::{Context, Result}; +use certbot::ChallengeKind; + +use crate::models::InstanceInfo; +use crate::time::{encode_ts, now_secs}; use serde::{Deserialize, Serialize}; use tokio::sync::watch; use wavekv::{node::NodeState, types::NodeId, Node}; @@ -64,9 +87,17 @@ pub struct PortPolicy { pub restrict_mode: bool, } -/// Instance core data (persistent) +/// What a CVM registered, stored at `inst/<instance_id>`. +/// +/// Every field here came from the CVM, which is what makes it safe for any node +/// to rewrite the whole record on any registration -- and what decides that an +/// operator's decisions do not live here. Those are separate keys; see +/// [`PortPolicyOverride`]. +/// +/// The assembled view the proxy routes on is [`crate::models::InstanceInfo`], +/// which is this plus those keys plus a live connection count. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct InstanceData { +pub struct InstanceRecord { pub app_id: String, pub ip: Ipv4Addr, pub public_key: String, @@ -81,12 +112,139 @@ pub struct InstanceData { /// the cache is invalidated and re-fetched lazily. #[serde(default)] pub port_policy_hash: String, - /// Operator-set override applied via the Admin RPC. Takes precedence over - /// the instance-reported `port_policy` when set, and survives app upgrades - /// (compose_hash changes do not clear it). Cleared explicitly via - /// ClearInstancePortPolicy. + /// Whether the CVM declared, at registration, that its traffic should be + /// gated on app health. Persisted so a restarted gateway (or a peer + /// learning this instance through sync) does not poll an app that never + /// opted in and read the failures as an unhealthy one. + /// + /// A declaration, so it belongs here with everything else the CVM said + /// about itself -- unlike the operator's gate, which any registration would + /// then be able to overwrite and which therefore has a key of its own. + /// + /// `Option` because `None` has to mean "the writer did not know about this + /// field" rather than "the app opted out": reading `false` off a record + /// written before the field existed would reset health to `Ungated` and + /// drop the app's cached selection on a record that never said so. A build + /// that predates the field round-trips it through + /// [`compat::carry_unknown_fields`], so `None` only reaches a reader after + /// a rollback and forward again. + #[serde(default)] + pub health_check: Option<bool>, +} + +/// What an operator has said about an instance's ports, under +/// `admin/<id>/port_policy`. +/// +/// Three answers, not two, which is why this is not an `Option<PortPolicy>`: +/// the key not existing is not the same as a key that says "cleared". Never set +/// falls back to the copy an older build left in the instance record; cleared +/// does not, or clearing an override would be undone by that copy. The wire +/// form is `Option<PortPolicy>` -- the key's existence carries the third state. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PortPolicyOverride { + /// An operator cleared it. + Cleared, + /// An operator set it. + Set(PortPolicy), +} + +/// The overrides an instance record still carries from before they had keys of +/// their own. +/// +/// `InstanceRecord` no longer declares these fields, which is deliberate on both +/// sides of an upgrade. Reading, this type recovers them for the move across. +/// Writing, an undeclared field is one [`compat::carry_unknown_fields`] +/// preserves verbatim, so a node still on the previous build keeps finding its +/// copy where it left it for as long as the upgrade takes. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)] +pub struct LegacyOverrides { #[serde(default)] pub admin_port_policy: Option<PortPolicy>, + #[serde(default)] + pub ready: Option<bool>, +} + +impl LegacyOverrides { + fn is_empty(&self) -> bool { + self.admin_port_policy.is_none() && self.ready.is_none() + } +} + +/// The record this node would publish for `info`. +/// +/// Every writer rewrites the whole record, so every writer has to name every +/// field -- and each hand-written copy is one more place a field added later +/// can be forgotten. `admin_port_policy` was dropped that way once already. +/// Going through one conversion makes the compiler the thing that remembers. +/// +/// What an operator set is deliberately absent: it lives under `admin/`, +/// written only from the Admin RPCs, so this rewrite -- which any node performs +/// on any registration -- cannot reach it. +impl From<&InstanceInfo> for InstanceRecord { + fn from(info: &InstanceInfo) -> Self { + Self { + app_id: info.app_id.clone(), + ip: info.ip, + public_key: info.public_key.clone(), + reg_time: encode_ts(info.reg_time), + port_policy: info.port_policy.clone(), + port_policy_hash: info.port_policy_hash.clone(), + // Only the declaration crosses the store. The verdict beside it in + // `Health` is this node's own observation, and a shared one would + // outlive the instance that earned it. + health_check: Some(info.health_check()), + } + } +} + +/// What one round of tombstone collection freed, per store. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CollectedTombstones { + pub persistent: usize, + pub ephemeral: usize, +} + +impl CollectedTombstones { + pub fn total(&self) -> usize { + self.persistent + self.ephemeral + } +} + +/// How many replicated writes this node covers, per store: the sum over every +/// origin of that origin's ack watermark. +/// +/// This is a function of replicated state, so every node's reading converges +/// within a sync round -- which is what lets the tombstone GC trigger on it +/// and land in the same window cluster-wide without any node reading a clock. +/// It is monotone in steady state, and steps back in two ways with different +/// lifetimes. Digest repair lowers an ack until retransmission restores it -- +/// a dip the GC trigger simply waits out. Removing a peer drops that origin's +/// watermark for good; the GC task answers by resetting its baseline (see +/// `start_tombstone_gc_task`) instead of waiting for the cluster to re-earn +/// writes it no longer remembers. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReplicatedWrites { + pub persistent: u64, + pub ephemeral: u64, +} + +/// The `inst/` records currently in the KV store, split by readability. +/// +/// A key that is absent or tombstoned does not appear here at all — that is the +/// signal that the instance was deleted. A key whose bytes no longer decode +/// lands in `undecodable`, which is deliberately *not* the same signal: the +/// record still exists, we just cannot read it, and dropping the instance from +/// the data plane on that basis would turn one unreadable record into an +/// outage. +#[derive(Debug, Default)] +pub struct LoadedInstances { + /// Records that decoded successfully, keyed by instance ID. + pub decoded: BTreeMap<String, InstanceRecord>, + /// Instance IDs whose stored bytes are present but no longer decode, + /// mapped to the decode error. Loading does not log these: the reload + /// path reports them on transitions, and read-only listings must stay + /// quiet no matter how often an operator runs them. + pub undecodable: BTreeMap<String, String>, } /// Gateway node status (stored separately for independent updates) @@ -221,6 +379,12 @@ pub struct ZtDomainConfig { /// The domain with highest priority is returned as the default base_domain in APIs #[serde(default)] pub priority: i32, + /// ACME challenge used to prove control of this domain. + /// + /// Records written before dns-persist-01 support have no such field and + /// decode as `dns-01`, which is the only method those deployments had. + #[serde(default)] + pub challenge: ChallengeKind, } /// Global certbot configuration (stored in KV, synced across nodes) @@ -237,6 +401,15 @@ pub struct GlobalCertbotConfig { pub renew_timeout: Duration, /// ACME server URL (None means use default Let's Encrypt production) pub acme_url: String, + /// Issuer Domain Name naming the CA in dns-persist-01 and CAA records. + /// + /// Empty means Let's Encrypt. A `dns-persist-01` record has to name a CA + /// the challenge lists in `issuer-domain-names`, and the CAA records written + /// for either challenge name the same CA, so a private or staging ACME + /// server needs its own value here -- one setting for both, since `acme_url` + /// is one setting for both. + #[serde(default)] + pub issuer_domain_name: String, } impl Default for GlobalCertbotConfig { @@ -246,15 +419,52 @@ impl Default for GlobalCertbotConfig { renew_before_expiration: Duration::from_secs(30 * 86400), // 30 days renew_timeout: Duration::from_secs(300), // 5 minutes acme_url: Default::default(), // default Let's Encrypt + issuer_domain_name: Default::default(), // default Let's Encrypt } } } +/// Durable record that an operator removed a node from the cluster (stored in +/// KV, synced across nodes). +/// +/// A **live** record, deliberately. The `__peer_addr` tombstone the removal +/// also writes is food for the tombstone GC, and a removal marker the +/// collector eventually eats reads as "never registered" at precisely the +/// moment the lockout matters -- after collection, when a returning node's +/// stale records have nothing left to beat them under LWW. A fact that must +/// outlive every tombstone cannot be expressed as a deletion. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub struct PeerRemovalRecord { + /// Unix seconds when the operator removed the node. + pub removed_at: u64, + /// The node that executed the removal. + pub removed_by: NodeId, +} + +/// Tombstone GC pacing (stored in KV, synced across nodes). +/// +/// The pace must be one number cluster-wide: nodes collecting on different +/// boundaries are back to collecting on their own phase, which the state +/// digest reads as divergence. Storing the override in the replicated KV is +/// what makes it one number; the per-node config file only supplies the +/// default for when this record is absent. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub struct GlobalTombstoneGcConfig { + /// Collect once every this many replicated writes, per store. Zero + /// disables collection cluster-wide. + pub writes_per_collection: u64, +} + // Key prefixes and builders pub mod keys { use super::NodeId; pub const INST_PREFIX: &str = "inst/"; + pub const ADMIN_PREFIX: &str = "admin/"; + /// Suffix of the traffic-gate key. See [`admin_ready`]. + pub const ADMIN_READY: &str = "ready"; + /// Suffix of the port-policy override key. See [`admin_port_policy`]. + pub const ADMIN_PORT_POLICY: &str = "port_policy"; pub const NODE_PREFIX: &str = "node/"; pub const NODE_INFO_PREFIX: &str = "node/info/"; pub const NODE_STATUS_PREFIX: &str = "node/status/"; @@ -262,17 +472,39 @@ pub mod keys { pub const HANDSHAKE_PREFIX: &str = "handshake/"; pub const LAST_SEEN_NODE_PREFIX: &str = "last_seen/node/"; pub const PEER_ADDR_PREFIX: &str = "__peer_addr/"; + pub const PEER_REMOVED_PREFIX: &str = "__peer_removed/"; pub const CERT_PREFIX: &str = "cert/"; pub const DNS_CRED_PREFIX: &str = "dns_cred/"; pub const DNS_CRED_DEFAULT: &str = "dns_cred_default"; + /// Shared by the `GLOBAL_*` keys below; not itself a key. + pub const GLOBAL_PREFIX: &str = "global/"; pub const GLOBAL_CERTBOT_CONFIG: &str = "global/certbot_config"; + pub const GLOBAL_TOMBSTONE_GC_CONFIG: &str = "global/tombstone_gc_config"; pub const GLOBAL_ACME_CREDENTIALS: &str = "global/acme_credentials"; pub const GLOBAL_ACME_ATTESTATION: &str = "global/acme_attestation"; + pub const GLOBAL_ACME_ROTATION_LOCK: &str = "global/acme_rotation_lock"; pub fn inst(instance_id: &str) -> String { format!("{INST_PREFIX}{instance_id}") } + /// Key for an instance's operator-set traffic gate. + /// + /// One key per override rather than one record holding both: WaveKV + /// resolves a conflict by taking a whole value, so two overrides sharing a + /// key means setting one on this node discards a peer's unsynced change to + /// the other. They are independent decisions and are set by independent + /// calls, so they get independent keys. + pub fn admin_ready(instance_id: &str) -> String { + format!("{ADMIN_PREFIX}{instance_id}/{ADMIN_READY}") + } + + /// Key for an instance's operator-set port-policy override. + /// See [`admin_ready`] for why this is not a field of the same record. + pub fn admin_port_policy(instance_id: &str) -> String { + format!("{ADMIN_PREFIX}{instance_id}/{ADMIN_PORT_POLICY}") + } + pub fn node_info(node_id: NodeId) -> String { format!("{NODE_INFO_PREFIX}{node_id}") } @@ -308,6 +540,10 @@ pub mod keys { format!("{PEER_ADDR_PREFIX}{node_id}") } + pub fn peer_removed(node_id: NodeId) -> String { + format!("{PEER_REMOVED_PREFIX}{node_id}") + } + // ==================== DNS Credential keys ==================== /// Key for a DNS credential @@ -366,8 +602,90 @@ pub mod keys { } } +/// How far into the future a replicated observation may be timestamped before +/// this node ignores it. +/// +/// `handshake/` and `last_seen/` records are wall-clock seconds written by +/// whichever node made the observation, and the gateway aggregates them with +/// `max`. Without a horizon, a single node with a fast clock — or one corrupt +/// record near `u64::MAX` — keeps a dead CVM "alive" on every node forever: +/// `recycle()` never fires and top-N routing keeps steering traffic at it. +/// 5 minutes is well above the drift between NTP-synced hosts and well below +/// the recycle timeout. +pub const MAX_CLOCK_DRIFT_SECS: u64 = 300; + +/// Drop observations timestamped beyond [`MAX_CLOCK_DRIFT_SECS`] into the +/// future, logging once per call with the number dropped. +fn drop_future_observations<T>( + observations: impl Iterator<Item = T>, + timestamp: impl Fn(&T) -> u64, + kind: &str, +) -> Vec<T> { + let horizon = now_secs().saturating_add(MAX_CLOCK_DRIFT_SECS); + let mut dropped = 0usize; + let kept = observations + .filter(|item| { + let plausible = timestamp(item) <= horizon; + dropped += usize::from(!plausible); + plausible + }) + .collect(); + if dropped > 0 { + warn!("ignored {dropped} {kind} observation(s) dated more than {MAX_CLOCK_DRIFT_SECS}s ahead of local time"); + } + kept +} + +/// Ceiling on a decompressed sync payload. +/// +/// The wire is gzipped, and gzip expands by three orders of magnitude on +/// attacker-chosen input: the 16 MiB cap on a request body is a cap on the *compressed* +/// size, which bounds nothing useful on its own. Every gateway in the cluster shares one +/// app_id, so mTLS proves only that a peer is *some* gateway of this deployment, and a +/// peer running a buggy build can send a body that decompresses to more memory than the +/// node has. +/// +/// The value is far above any legitimate payload. A v2 delta is capped by +/// `max_delta_bytes` (4 MiB by default). +pub const MAX_DECOMPRESSED_SYNC_BYTES: usize = 128 * 1024 * 1024; + +/// Ceiling on a compressed sync response, mirroring the 16 MiB the routes accept on a +/// request. Without it a peer's response body is read to completion before any decoding +/// bound applies. +pub const MAX_COMPRESSED_SYNC_BYTES: usize = 16 * 1024 * 1024; + +/// Decompress gzip, refusing anything that expands past `limit`. +/// +/// Reads one byte past the limit so a payload landing exactly on it is still accepted +/// and a larger one is rejected rather than silently truncated — `Read::take` alone +/// would hand back a short buffer that then fails to decode, reporting the wrong fault. +pub fn gunzip_bounded(data: &[u8], limit: usize) -> Result<Vec<u8>> { + use std::io::Read; + + let mut out = Vec::new(); + flate2::read::GzDecoder::new(data) + .take(limit as u64 + 1) + .read_to_end(&mut out) + .context("failed to decompress payload")?; + if out.len() > limit { + anyhow::bail!("decompressed payload exceeds {limit} bytes"); + } + Ok(out) +} + +/// Encode a KV value as MessagePack. +/// +/// Structs are encoded as maps keyed by field name rather than as positional +/// arrays. Field-name keys let a reader skip fields it does not know and fill +/// in `#[serde(default)]` fields it does not receive, so the value types below +/// can gain fields without breaking gateways running an older build. Decoding +/// accepts both forms, so values written by older releases stay readable. +/// +/// Skipping on read is only half of a rolling upgrade: this encoding contains +/// exactly the fields the writing binary declares, so an older node rewriting a +/// record would drop the rest. [`compat`] puts them back on the way out. pub fn encode<T: Serialize>(value: &T) -> Result<Vec<u8>> { - rmp_serde::encode::to_vec(value).context("failed to encode value") + rmp_serde::encode::to_vec_named(value).context("failed to encode value") } pub fn decode<T: for<'de> Deserialize<'de>>(bytes: &[u8]) -> Result<T> { @@ -376,7 +694,13 @@ pub fn decode<T: for<'de> Deserialize<'de>>(bytes: &[u8]) -> Result<T> { trait GetPutCodec { fn decode<T: for<'de> serde::Deserialize<'de>>(&self, key: &str) -> Option<T>; - fn put_encoded<T: serde::Serialize>(&mut self, key: String, value: &T) -> Result<()>; + fn decode_strict<T: for<'de> serde::Deserialize<'de>>(&self, key: &str) -> Result<Option<T>>; + fn put_encoded<T: serde::Serialize + serde::de::DeserializeOwned>( + &mut self, + key: String, + value: &T, + update: bool, + ) -> Result<()>; fn iter_decoded<T: for<'de> serde::Deserialize<'de>>( &self, prefix: &str, @@ -385,6 +709,10 @@ trait GetPutCodec { &self, prefix: &str, ) -> impl Iterator<Item = T>; + fn iter_decoded_strict<T: for<'de> serde::Deserialize<'de>>( + &self, + prefix: &str, + ) -> impl Iterator<Item = (String, Result<T>)>; } impl GetPutCodec for NodeState { @@ -393,14 +721,58 @@ impl GetPutCodec for NodeState { .and_then(|entry| match decode(entry.value.as_ref()?) { Ok(value) => Some(value), Err(e) => { + crate::metrics::record_decode_failure(key); warn!("failed to decode value for key {key}: {e:?}"); None } }) } - fn put_encoded<T: serde::Serialize>(&mut self, key: String, value: &T) -> Result<()> { - self.put(key.clone(), encode(value)?) + /// Three-state read: `Ok(None)` for a key that is missing or tombstoned, + /// `Ok(Some)` for a decodable value, `Err` for a stored value that no + /// longer decodes. + /// + /// [`Self::decode`] folds corruption into `None`, which is right for + /// per-instance records (skip the bad one, keep serving the rest) and + /// wrong for global records, where "absent" means "apply the default" and + /// a corrupt record would silently change cluster-wide behavior. + fn decode_strict<T: for<'de> serde::Deserialize<'de>>(&self, key: &str) -> Result<Option<T>> { + let Some(entry) = self.get(key) else { + return Ok(None); + }; + // A `None` value is a tombstone: the key was deliberately deleted. + let Some(value) = entry.value.as_ref() else { + return Ok(None); + }; + decode(value) + .map(Some) + .with_context(|| format!("corrupt record at KV key {key}")) + } + + /// Write a value. + /// + /// `update` is true when the write modifies a record that outlives it, and + /// false when it states a complete new fact. An update keeps the fields the + /// stored record holds and this binary does not declare; a replacement does + /// not, because they describe the fact being replaced. See [`compat`] for + /// why that is the caller's call, and why keeping a field is not the same + /// as keeping every field the encoding happens to be missing. + fn put_encoded<T: serde::Serialize + serde::de::DeserializeOwned>( + &mut self, + key: String, + value: &T, + update: bool, + ) -> Result<()> { + let encoded = encode(value)?; + let declared = update.then(compat::declared_fields::<T>).flatten(); + let bytes = match declared { + Some(declared) => match self.get(&key).and_then(|entry| entry.value) { + Some(stored) => compat::carry_unknown_fields::<T>(&key, declared, &stored, encoded), + None => encoded, + }, + None => encoded, + }; + self.put(key.clone(), bytes) .with_context(|| format!("failed to put key {key}"))?; Ok(()) } @@ -413,6 +785,7 @@ impl GetPutCodec for NodeState { let value = match decode(entry.value.as_ref()?) { Ok(value) => value, Err(e) => { + crate::metrics::record_decode_failure(key); warn!("failed to decode value for key {key}: {e:?}"); return None; } @@ -429,6 +802,7 @@ impl GetPutCodec for NodeState { let value = match decode(entry.value.as_ref()?) { Ok(value) => value, Err(e) => { + crate::metrics::record_decode_failure(key); warn!("failed to decode value for key {key}: {e:?}"); return None; } @@ -436,6 +810,73 @@ impl GetPutCodec for NodeState { Some(value) }) } + + /// Like [`Self::iter_decoded`], but surfaces undecodable records instead of + /// skipping them. + /// + /// Tombstoned keys are still skipped — a deleted record and an unreadable + /// one call for opposite responses, and only this form lets the caller tell + /// them apart. + fn iter_decoded_strict<T: for<'de> serde::Deserialize<'de>>( + &self, + prefix: &str, + ) -> impl Iterator<Item = (String, Result<T>)> { + self.iter_by_prefix(prefix).filter_map(|(key, entry)| { + let value = entry.value.as_ref()?; + Some(( + key.to_string(), + decode(value).with_context(|| format!("corrupt record at KV key {key}")), + )) + }) + } +} + +/// Which of an instance's five keys a write is touching. +/// +/// Named at every call site because the delete path issues all of them and +/// reports them individually: an operator can do something about an `inst/` +/// tombstone that did not land and nothing about an ephemeral observation that +/// did not, so "a delete failed" is not a useful thing to be told. +/// +/// It is also what `KvStore::fail_writes_for_test` aims at, which is why the +/// write paths take one instead of a `u8` that means nothing outside a test +/// build. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum InstanceKey { + /// The persistent `inst/` record, and its tombstone. + Instance, + /// The persistent `admin/<id>/ready` record, and its tombstone. + Gate, + /// The persistent `admin/<id>/port_policy` record, and its tombstone. + PortPolicyOverride, + /// The ephemeral `conn/` key this node owns. + Connections, + /// The ephemeral `handshake/` key this node owns. + Handshake, +} + +impl InstanceKey { + /// What an error message calls it. + fn name(self) -> &'static str { + match self { + Self::Instance => "instance", + Self::Gate => "traffic gate", + Self::PortPolicyOverride => "port-policy override", + Self::Connections => "connection count", + Self::Handshake => "handshake observation", + } + } + + #[cfg(test)] + fn bit(self) -> u8 { + match self { + Self::Instance => 0b00001, + Self::Gate => 0b00010, + Self::PortPolicyOverride => 0b00100, + Self::Connections => 0b01000, + Self::Handshake => 0b10000, + } + } } /// Sync store wrapping two WaveKV Nodes (persistent and ephemeral). @@ -450,18 +891,121 @@ pub struct KvStore { ephemeral: Node, /// This gateway's node ID my_node_id: NodeId, + /// Which instance writes to fail, so the paths that report a failure can + /// be tested at all. + /// + /// The real failures here are WAL I/O -- no space, permission denied, the + /// data volume unmounted -- and none of them can be provoked from a test + /// without a filesystem to sabotage. Faking the outcome is the only way to + /// cover the code that reports them, which is the whole reason + /// `persist_instance_record` returns a `Result` and the whole reason + /// `sync_delete_instance` does not short-circuit. + /// + /// A bitmask of [`FailWrite`]. + #[cfg(test)] + injected_failures: std::sync::Arc<std::sync::atomic::AtomicU8>, +} + +/// Whether opening the persistent store failed because the storage is +/// unavailable, rather than because the stored bytes are unreadable. +/// +/// wavekv reports both through `anyhow`, so they have to be told apart by what +/// is in the error chain. Unreadable content arrives as a decode failure, a +/// checksum or header `bail!`, or a read that ran off the end of a truncated +/// file — the last of which is an `io::Error`, but only ever `UnexpectedEof` or +/// `InvalidData`. Every other `io::Error` is the storage layer talking: no +/// space left, permission denied, too many open files, the data volume not +/// mounted yet. +fn is_storage_failure(err: &anyhow::Error) -> bool { + err.chain() + .filter_map(|cause| cause.downcast_ref::<std::io::Error>()) + .any(|io| { + !matches!( + io.kind(), + std::io::ErrorKind::UnexpectedEof | std::io::ErrorKind::InvalidData + ) + }) } impl KvStore { - /// Create a new sync store + /// Create a new sync store. + /// + /// If the on-disk WAL/snapshot cannot be *read*, the data directory is + /// moved aside and the store starts empty rather than refusing to boot: the + /// persistent state is replicated on every peer, a torn WAL tail is the + /// normal artifact of a crash, and a gateway that cannot start serves no + /// traffic at all. Nothing is deleted — the unreadable directory is kept + /// under `<data_dir>.corrupt.<unix_ts>` for inspection. + /// + /// A failure of the *storage* is a different matter and fails the boot. A + /// full disk, an exhausted fd table or a volume that has not finished + /// mounting all say nothing about the contents, so moving the directory + /// aside would discard intact state — and, because the condition persists + /// across restarts, would do it again on every attempt, burying the real + /// data under a pile of `.corrupt.*` directories. Failing here instead + /// leaves the state alone and puts the actual cause in front of the + /// operator, which for a single-node deployment holding the only copy of + /// the ACME account and DNS credentials is the difference between a restart + /// and a rebuild. + /// + /// `wal_sync_window` is how long a write may sit in the page cache before + /// the log is forced to disk; `None` forces every write before it returns. + /// It applies only to the persistent store — the ephemeral one keeps no log + /// and never touches the disk. pub fn new( my_node_id: NodeId, peer_ids: Vec<NodeId>, data_dir: impl AsRef<Path>, + wal_sync_window: Option<Duration>, ) -> Result<Self> { - let persistent = - Node::new_with_persistence(my_node_id, peer_ids.clone(), data_dir.as_ref()) - .context("failed to create persistent wavekv node")?; + let data_dir = data_dir.as_ref(); + let node_config = wavekv::NodeConfig { + // wavekv still calls it an interval; the gateway calls it a window + // because zero here means "no window", not "off". + wal_sync_interval: wal_sync_window, + ..Default::default() + }; + let persistent = match Node::with_persistence_and_config( + my_node_id, + peer_ids.clone(), + data_dir, + node_config.clone(), + ) { + Ok(node) => node, + Err(err) if is_storage_failure(&err) => { + return Err(err).with_context(|| { + format!( + "cannot open the WaveKV data dir {}; refusing to start rather than \ + quarantine a directory whose contents are most likely intact", + data_dir.display() + ) + }); + } + Err(err) => { + // Keep the original open error in the context: if moving the + // directory aside also fails, the reason the open failed is the + // more useful half of the diagnosis and is otherwise lost. + let quarantined = quarantine_data_dir(data_dir).with_context(|| { + format!( + "failed to open the WaveKV data dir ({err:#}) and failed to \ + move it aside for recovery" + ) + })?; + error!( + "WaveKV data dir {} is unreadable ({err:#}); moved it to {} and started empty — \ + state will be re-fetched from peers", + data_dir.display(), + quarantined.display(), + ); + Node::with_persistence_and_config( + my_node_id, + peer_ids.clone(), + data_dir, + node_config, + ) + .context("failed to create persistent wavekv node on a fresh data dir")? + } + }; // Get peers from persistent store (may have been restored from WAL) // and include them when creating ephemeral store @@ -479,6 +1023,8 @@ impl KvStore { persistent, ephemeral, my_node_id, + #[cfg(test)] + injected_failures: Default::default(), }) } @@ -497,44 +1043,311 @@ impl KvStore { // ==================== Instance Sync ==================== /// Sync instance data to other nodes - pub fn sync_instance(&self, instance_id: &str, data: &InstanceData) -> Result<()> { + pub fn sync_instance(&self, instance_id: &str, data: &InstanceRecord) -> Result<()> { + self.injected_failure(InstanceKey::Instance)?; self.persistent .write() - .put_encoded(keys::inst(instance_id), data) + .put_encoded(keys::inst(instance_id), data, true) } - /// Sync instance deletion to other nodes - pub fn sync_delete_instance(&self, instance_id: &str) -> Result<()> { - self.persistent.write().delete(keys::inst(instance_id))?; - self.ephemeral - .write() - .delete(keys::conn(instance_id, self.my_node_id))?; - // Delete this node's handshake record - self.ephemeral - .write() - .delete(keys::handshake(instance_id, self.my_node_id))?; - Ok(()) + // ==================== Operator Override Sync ==================== + + /// The operator's traffic gate for one instance. `None` means no key. + /// + /// An unreadable record is an error, not a `None`. Reading it as "nobody + /// gated this" is the one answer that must not be guessed: it returns a + /// quarantined instance to rotation. + pub fn instance_gate(&self, instance_id: &str) -> Result<Option<bool>> { + self.persistent + .read() + .decode_strict::<bool>(&keys::admin_ready(instance_id)) + } + + /// The operator's port-policy override for one instance. `None` means no + /// key -- which is not [`PortPolicyOverride::Cleared`]. + pub fn instance_port_policy_override( + &self, + instance_id: &str, + ) -> Result<Option<PortPolicyOverride>> { + let stored = self + .persistent + .read() + .decode_strict::<Option<PortPolicy>>(&keys::admin_port_policy(instance_id))?; + Ok(stored.map(|policy| match policy { + Some(policy) => PortPolicyOverride::Set(policy), + None => PortPolicyOverride::Cleared, + })) } - /// Load all instances from sync store (for initial sync on startup) - pub fn load_all_instances(&self) -> BTreeMap<String, InstanceData> { + /// Publish an instance's operator-set traffic gate. + pub fn set_instance_gate(&self, instance_id: &str, ready: bool) -> Result<()> { + self.put_admin_override(InstanceKey::Gate, keys::admin_ready(instance_id), &ready) + } + + /// Publish an instance's operator-set port-policy override. `None` clears it. + /// + /// A cleared override is written, not deleted: an absent key is what lets + /// the copy an older build left in the instance record apply, so deleting + /// would put the override straight back. + pub fn set_instance_port_policy_override( + &self, + instance_id: &str, + policy: Option<PortPolicy>, + ) -> Result<()> { + self.put_admin_override( + InstanceKey::PortPolicyOverride, + keys::admin_port_policy(instance_id), + &policy, + ) + } + + /// The overrides an instance record still carries from before they had keys + /// of their own. + /// + /// Read from the same `inst/` bytes as [`InstanceRecord`], through a type + /// that declares only the retired fields. Bulk rather than per instance + /// because the only caller wants to know whether *any* remain. + pub fn legacy_instance_overrides(&self) -> BTreeMap<String, LegacyOverrides> { self.persistent .read() - .iter_decoded(keys::INST_PREFIX) - .filter_map(|(key, data)| { - let instance_id = keys::parse_inst_key(&key)?; - Some((instance_id.into(), data)) + .iter_decoded::<LegacyOverrides>(keys::INST_PREFIX) + .filter_map(|(key, legacy)| { + if legacy.is_empty() { + return None; + } + Some((keys::parse_inst_key(&key)?.to_string(), legacy)) }) .collect() } + /// Move any overrides still living in an instance record to their own keys. + /// + /// Runs on every reload rather than once at startup, because the source is + /// not only this node's own data directory: a node still on the previous + /// build writes overrides into `inst/`, and this is how they reach the new + /// keys during a rolling upgrade instead of being lost at that instance's + /// next re-registration. + /// + /// Per override, not per instance -- which is the point of them having a + /// key each. Only where the key does not exist at all: once it does it is + /// the answer, including when it says "no override", which is what clearing + /// one leaves behind. Without that rule, clearing an override here would be + /// undone by the stale copy an old node left in the instance record. An + /// unreadable key counts as existing for the same reason. + /// + /// Returns the keys written. Failures are logged and skipped: the next + /// reload tries again, and the read-side fallback keeps the override in + /// force in the meantime. + pub fn migrate_legacy_instance_overrides(&self) -> Vec<String> { + let mut moved = Vec::new(); + for (instance_id, legacy) in self.legacy_instance_overrides() { + if let Some(ready) = legacy.ready { + if matches!(self.instance_gate(&instance_id), Ok(None)) { + self.record_migration( + &mut moved, + InstanceKey::Gate, + keys::admin_ready(&instance_id), + &ready, + ); + } + } + if legacy.admin_port_policy.is_some() + && matches!(self.instance_port_policy_override(&instance_id), Ok(None)) + { + self.record_migration( + &mut moved, + InstanceKey::PortPolicyOverride, + keys::admin_port_policy(&instance_id), + &legacy.admin_port_policy, + ); + } + } + moved + } + + fn record_migration<T: Serialize + serde::de::DeserializeOwned>( + &self, + moved: &mut Vec<String>, + record: InstanceKey, + key: String, + value: &T, + ) { + match self.put_admin_override(record, key.clone(), value) { + Ok(()) => moved.push(key), + Err(err) => warn!("failed to move operator override to {key}: {err:?}"), + } + } + + /// Written as a replacement rather than an update: each record states one + /// complete fact, so carrying a field a peer wrote would leave half of one + /// operator's decision beside half of another's. + fn put_admin_override<T: Serialize + serde::de::DeserializeOwned>( + &self, + record: InstanceKey, + key: String, + value: &T, + ) -> Result<()> { + self.injected_failure(record) + .and_then(|()| self.persistent.write().put_encoded(key, value, false)) + .with_context(|| format!("failed to write the {} record", record.name())) + } + + /// Fail every write touching one of `records` until called again. See + /// [`KvStore::injected_failures`]. + #[cfg(test)] + pub(crate) fn fail_writes_for_test(&self, records: &[InstanceKey]) { + let mask = records.iter().fold(0u8, |mask, record| mask | record.bit()); + self.injected_failures + .store(mask, std::sync::atomic::Ordering::Relaxed); + } + + /// Deliberately terse: what an operator reads is the context the write path + /// adds, so a test that asserts on the message is asserting on the real one. + #[cfg(test)] + fn injected_failure(&self, record: InstanceKey) -> Result<()> { + let mask = self + .injected_failures + .load(std::sync::atomic::Ordering::Relaxed); + anyhow::ensure!(mask & record.bit() == 0, "injected store failure"); + Ok(()) + } + + /// Nothing to inject outside a test build, so the write paths read the same + /// in both instead of carrying a `cfg` in their bodies. + #[cfg(not(test))] + #[inline(always)] + fn injected_failure(&self, _record: InstanceKey) -> Result<()> { + Ok(()) + } + + /// Sync instance deletion to other nodes + /// + /// Returns whether a live record (including an undecodable one) existed + /// before the tombstone was written. + /// + /// All five deletes are issued unconditionally, including when no `inst/` + /// record was there to tombstone. Re-issuing removal is the only way to + /// clean up records orphaned by an earlier partial failure. + /// + /// Only failure to tombstone `inst/` is returned. Failures deleting the + /// associated overrides or this node's ephemeral observations are logged -- + /// even when the tombstone also failed: they do not change whether the CVM + /// was removed, and must not prevent the local routing and WireGuard + /// cleanup that follows this call. + pub fn sync_delete_instance(&self, instance_id: &str) -> Result<bool> { + // Every delete is attempted even if an earlier one fails. Short-circuiting + // on `?` would leave the later keys behind while the `inst/` tombstone is + // already published, and nothing garbage-collects orphans. + let previous = self.delete_persistent(keys::inst(instance_id), InstanceKey::Instance); + // The overrides outlive the instance record unless they are tombstoned + // too, and instance ids are recycled -- an id reused after a delete + // would inherit the gate and port policy an operator set for whatever + // ran under it before. + let gate = self.delete_persistent(keys::admin_ready(instance_id), InstanceKey::Gate); + let override_policy = self.delete_persistent( + keys::admin_port_policy(instance_id), + InstanceKey::PortPolicyOverride, + ); + let observations = self.sync_forget_local_observations(instance_id); + + // Logged before `previous` can return: a store-wide fault fails these + // together with the `inst/` tombstone, and that compound failure is + // exactly when the operator needs the inventory of what was left + // behind. An orphaned override is a persistent record a recycled id + // would inherit, so it is an error; a leaked observation is inert + // telemetry, so it is a warning -- matching the reload path. + for result in [gate, override_policy] { + if let Err(err) = result { + error!( + "failed to delete an override while removing instance \ + {instance_id}, leaving an orphan record: {err:?}" + ); + } + } + if let Err(err) = observations { + warn!( + "failed to delete local observations while removing instance \ + {instance_id}, leaving orphan telemetry records: {err:?}" + ); + } + Ok(previous?.is_some_and(|entry| !entry.is_deleted())) + } + + /// Withdraw this node's observations of an instance. + /// + /// `conn/` and `handshake/` are keyed by observer, so a delete can only + /// withdraw the deleting node's own -- they are the only ones it owns. + /// Every other node has to withdraw its own when it learns of the deletion, + /// or its observation outlives the CVM it describes: nothing sweeps those + /// prefixes, and a node's recycle loop cannot see an instance it has + /// already dropped from memory. They are ephemeral, so a restart collects + /// them, but a gateway's uptime is measured in months. + pub fn sync_forget_local_observations(&self, instance_id: &str) -> Result<()> { + let conn = self.delete_ephemeral( + keys::conn(instance_id, self.my_node_id), + InstanceKey::Connections, + ); + let handshake = self.delete_ephemeral( + keys::handshake(instance_id, self.my_node_id), + InstanceKey::Handshake, + ); + conn?; + handshake?; + Ok(()) + } + + fn delete_persistent( + &self, + key: String, + record: InstanceKey, + ) -> Result<Option<wavekv::types::Entry>> { + self.injected_failure(record) + .and_then(|()| self.persistent.write().delete(key)) + .with_context(|| format!("failed to delete the {} record", record.name())) + } + + fn delete_ephemeral( + &self, + key: String, + record: InstanceKey, + ) -> Result<Option<wavekv::types::Entry>> { + self.injected_failure(record) + .and_then(|()| self.ephemeral.write().delete(key)) + .with_context(|| format!("failed to delete the {} record", record.name())) + } + + /// Load all instances from the sync store. + pub fn load_all_instances(&self) -> LoadedInstances { + let mut loaded = LoadedInstances::default(); + for (key, result) in self + .persistent + .read() + .iter_decoded_strict::<InstanceRecord>(keys::INST_PREFIX) + { + let Some(instance_id) = keys::parse_inst_key(&key) else { + continue; + }; + match result { + Ok(data) => { + loaded.decoded.insert(instance_id.into(), data); + } + Err(err) => { + loaded + .undecodable + .insert(instance_id.into(), format!("{err:#}")); + } + } + } + loaded + } + // ==================== Node Sync ==================== /// Sync node data to other nodes pub fn sync_node(&self, node_id: NodeId, data: &NodeData) -> Result<()> { self.persistent .write() - .put_encoded(keys::node_info(node_id), data) + .put_encoded(keys::node_info(node_id), data, true) } /// Load all nodes from sync store @@ -549,13 +1362,42 @@ impl KvStore { .collect() } + /// Remove a gateway node from replicated state. + /// + /// Writes tombstones for the node's info, status, and sync address, and + /// drops this node's own last_seen observation of it. The `__peer_addr` + /// tombstone doubles as the cluster-wide removal signal: every gateway + /// prunes its sync peer set when it observes the deletion (see + /// [`Self::prune_removed_peers`]). + /// + /// Returns whether any of the node's persistent records was live before + /// the tombstones were written, so a node known only by its sync address + /// (registered via `SetNodeUrl` but never booted) still reports as + /// existing. + pub fn sync_remove_node(&self, node_id: NodeId) -> Result<bool> { + let previous = { + let mut persistent = self.persistent.write(); + [ + persistent.delete(keys::node_info(node_id))?, + persistent.delete(keys::node_status(node_id))?, + persistent.delete(keys::peer_addr(node_id))?, + ] + }; + self.ephemeral + .write() + .delete(keys::last_seen_node(node_id, self.my_node_id))?; + Ok(previous + .into_iter() + .any(|entry| entry.is_some_and(|entry| !entry.is_deleted()))) + } + // ==================== Node Status Sync ==================== /// Set node status (stored separately from NodeData) pub fn set_node_status(&self, node_id: NodeId, status: NodeStatus) -> Result<()> { self.persistent .write() - .put_encoded(keys::node_status(node_id), &status)?; + .put_encoded(keys::node_status(node_id), &status, false)?; Ok(()) } @@ -579,13 +1421,41 @@ impl KvStore { .collect() } + /// Whether a node counts as active. A node with no recorded status is up. + /// + /// The routing path and the metrics sampler both filter on this, and they + /// have to agree: a gauge that counts a node the router has dropped is + /// describing a routing table that does not exist. + pub(crate) fn node_is_active(status: Option<&NodeStatus>) -> bool { + !matches!(status, Some(NodeStatus::Down)) + } + + /// Count all and active nodes, without materialising `GatewayNodeInfo`. + /// + /// A scrape wants two numbers. Reaching them through `get_all_nodes()` and + /// `get_active_nodes()` instead means loading the node table twice, cloning + /// five strings per node, and taking the ephemeral lock once per node for a + /// `last_seen` that the count never reads -- all of it under the proxy lock + /// that the data path takes on every connection. + pub fn count_nodes(&self) -> (u64, u64) { + let statuses = self.load_all_node_statuses(); + let nodes = self.load_all_nodes(); + let active = nodes + .keys() + .filter(|id| Self::node_is_active(statuses.get(id))) + .count() as u64; + (nodes.len() as u64, active) + } + // ==================== Connection Count Sync ==================== /// Sync connection count for an instance (from this node) pub fn sync_connections(&self, instance_id: &str, count: u64) -> Result<()> { - self.ephemeral - .write() - .put_encoded(keys::conn(instance_id, self.my_node_id), &count)?; + self.ephemeral.write().put_encoded( + keys::conn(instance_id, self.my_node_id), + &count, + false, + )?; Ok(()) } @@ -593,15 +1463,21 @@ impl KvStore { /// Sync handshake timestamp for an instance (as observed by this node) pub fn sync_instance_handshake(&self, instance_id: &str, timestamp: u64) -> Result<()> { - self.ephemeral - .write() - .put_encoded(keys::handshake(instance_id, self.my_node_id), ×tamp)?; + self.ephemeral.write().put_encoded( + keys::handshake(instance_id, self.my_node_id), + ×tamp, + false, + )?; Ok(()) } - /// Get all handshake observations for an instance (from all nodes) + /// Get all handshake observations for an instance (from all nodes). + /// + /// Observations dated into the future are dropped; see + /// [`MAX_CLOCK_DRIFT_SECS`]. pub fn get_instance_handshakes(&self, instance_id: &str) -> BTreeMap<NodeId, u64> { - self.ephemeral + let observations = self + .ephemeral .read() .iter_decoded(&keys::handshake_prefix(instance_id)) .filter_map(|(key, ts)| { @@ -609,28 +1485,39 @@ impl KvStore { let observer: NodeId = suffix.parse().ok()?; Some((observer, ts)) }) + .collect::<Vec<_>>(); + drop_future_observations(observations.into_iter(), |(_, ts)| *ts, "handshake") + .into_iter() .collect() } - /// Get the latest handshake timestamp for an instance (max across all nodes) + /// Get the latest handshake timestamp for an instance (max across all + /// nodes), ignoring future-dated observations. pub fn get_instance_latest_handshake(&self, instance_id: &str) -> Option<u64> { - self.ephemeral + let observations = self + .ephemeral .read() .iter_decoded_values(&keys::handshake_prefix(instance_id)) + .collect::<Vec<u64>>(); + drop_future_observations(observations.into_iter(), |ts| *ts, "handshake") + .into_iter() .max() } /// Sync node last_seen (as observed by this node) pub fn sync_node_last_seen(&self, node_id: NodeId, timestamp: u64) -> Result<()> { - self.ephemeral - .write() - .put_encoded(keys::last_seen_node(node_id, self.my_node_id), ×tamp)?; + self.ephemeral.write().put_encoded( + keys::last_seen_node(node_id, self.my_node_id), + ×tamp, + false, + )?; Ok(()) } - /// Get all observations of a node's last_seen + /// Get all observations of a node's last_seen, ignoring future-dated ones. pub fn get_node_last_seen_by_all(&self, node_id: NodeId) -> BTreeMap<NodeId, u64> { - self.ephemeral + let observations = self + .ephemeral .read() .iter_decoded(&keys::last_seen_node_prefix(node_id)) .filter_map(|(key, ts)| { @@ -638,14 +1525,22 @@ impl KvStore { let seen_by: NodeId = suffix.parse().ok()?; Some((seen_by, ts)) }) + .collect::<Vec<_>>(); + drop_future_observations(observations.into_iter(), |(_, ts)| *ts, "node last_seen") + .into_iter() .collect() } - /// Get the latest last_seen timestamp for a node (max across all observers) + /// Get the latest last_seen timestamp for a node (max across all + /// observers), ignoring future-dated observations. pub fn get_node_latest_last_seen(&self, node_id: NodeId) -> Option<u64> { - self.ephemeral + let observations = self + .ephemeral .read() .iter_decoded_values(&keys::last_seen_node_prefix(node_id)) + .collect::<Vec<u64>>(); + drop_future_observations(observations.into_iter(), |ts| *ts, "node last_seen") + .into_iter() .max() } @@ -656,17 +1551,130 @@ impl KvStore { self.persistent.watch_prefix(keys::INST_PREFIX) } + /// Watch for changes to any instance's operator-set overrides. + /// + /// A separate watcher because they are a separate key: a peer opening or + /// closing a traffic gate touches `admin/` and nothing else, so the + /// instance watcher never fires and the gate would sit in the store + /// unapplied on every node but the one that took the RPC. + pub fn watch_instance_overrides(&self) -> watch::Receiver<()> { + self.persistent.watch_prefix(keys::ADMIN_PREFIX) + } + /// Watch for remote node changes pub fn watch_nodes(&self) -> watch::Receiver<()> { self.persistent.watch_prefix(keys::NODE_PREFIX) } + /// Watch for changes to replicated peer sync addresses + pub fn watch_peer_addrs(&self) -> watch::Receiver<()> { + self.persistent.watch_prefix(keys::PEER_ADDR_PREFIX) + } + // ==================== Persistence ==================== + /// Drop tombstones every known peer has already covered. + /// + /// wavekv gates this on an ack watermark rather than a clock: a tombstone + /// may go only once every peer reports having seen the delete that wrote + /// it. v1 took a TTL instead and 2.0 removed that API, because under any + /// state-shipping scheme an uncoordinated clock-based collection lets a + /// lagging replica resurrect the key -- here, a deregistered CVM + /// reappearing in every node's WireGuard config. + /// + /// Both stores, because both accumulate. A `conn/` or `handshake/` + /// tombstone is only in memory and goes on restart; an `inst/` or `admin/` + /// one is on disk and does not, so without this it is kept for the life of + /// the deployment. + /// + /// Both are attempted even if the first fails, for the same reason the + /// deletes are: stopping halfway leaves the stores disagreeing about what + /// has been collected, which is the state the digest reports as divergence. + /// And when both fail, the error names both -- the second failure is a + /// separate fact, not a detail of the first. + pub fn collect_tombstone_garbage(&self) -> Result<CollectedTombstones> { + let persistent = self + .persistent + .write() + .collect_tombstone_garbage() + .context("failed to collect tombstones from the persistent store"); + let ephemeral = self + .ephemeral + .write() + .collect_tombstone_garbage() + .context("failed to collect tombstones from the ephemeral store"); + match (persistent, ephemeral) { + (Ok(persistent), Ok(ephemeral)) => Ok(CollectedTombstones { + persistent, + ephemeral, + }), + (Err(err), Ok(_)) | (Ok(_), Err(err)) => Err(err), + (Err(persistent), Err(ephemeral)) => { + Err(persistent.context(format!("the ephemeral store failed too: {ephemeral:#}"))) + } + } + } + + /// How many replicated writes this node covers, per store. + /// + /// The tombstone GC trigger compares this against a boundary; see + /// `ReplicatedWrites` for why it is a count and not a clock. + pub fn replicated_writes(&self) -> ReplicatedWrites { + let sum = |node: &Node| { + node.read() + .acks_snapshot() + .values() + .fold(0u64, |acc, seq| acc.saturating_add(*seq)) + }; + ReplicatedWrites { + persistent: sum(&self.persistent), + ephemeral: sum(&self.ephemeral), + } + } + + /// The sync peer set (both stores share membership). + /// + /// The tombstone GC task watches this for shrinkage: a removed peer takes + /// its ack watermark out of `replicated_writes` permanently, which must + /// not be read as "the cluster has stopped writing". + pub fn peer_ids(&self) -> BTreeSet<NodeId> { + self.persistent.read().get_peers().into_iter().collect() + } + + /// Get the tombstone GC pacing override, if an operator has stored one. + /// + /// `None` means "use the config-file default". Fails closed on a corrupt + /// record: falling back to either the default or `None` would silently put + /// this node on a different collection boundary from its peers. + pub fn get_tombstone_gc_config(&self) -> Result<Option<GlobalTombstoneGcConfig>> { + self.persistent + .read() + .decode_strict(keys::GLOBAL_TOMBSTONE_GC_CONFIG) + } + + /// Store the tombstone GC pacing override, replicating it to every node. + pub fn set_tombstone_gc_config(&self, config: &GlobalTombstoneGcConfig) -> Result<()> { + self.persistent.write().put_encoded( + keys::GLOBAL_TOMBSTONE_GC_CONFIG.to_string(), + config, + true, + )?; + Ok(()) + } + pub fn persist_if_dirty(&self) -> Result<bool> { self.persistent.persist_if_dirty() } + /// Force the write-ahead log to disk if the configured window has elapsed. + /// + /// Returns whether an fsync happened. A no-op when no window is configured + /// — every write was already forced — or when nothing has been written + /// since the last one, so an idle gateway costs a lock acquisition. + pub fn sync_wal_if_due(&self) -> Result<bool> { + self.persistent.sync_wal_if_due() + } + // ==================== Peer Management ==================== pub fn add_peer(&self, peer_id: NodeId) -> Result<()> { @@ -675,6 +1683,107 @@ impl KvStore { Ok(()) } + /// Drop a node from the sync peer set of both stores. + /// + /// Returns whether the persistent store still had it as a peer. + pub fn remove_peer(&self, peer_id: NodeId) -> Result<bool> { + let removed = self.persistent.write().remove_peer(peer_id)?; + self.ephemeral.write().remove_peer(peer_id)?; + Ok(removed) + } + + /// Mark a node as removed by an operator. + /// + /// A live record rather than a tombstone -- see [`PeerRemovalRecord`] for + /// why the durable fact cannot be the `__peer_addr` deletion itself. + pub fn mark_peer_removed(&self, node_id: NodeId) -> Result<()> { + let record = PeerRemovalRecord { + removed_at: now_secs(), + removed_by: self.my_node_id, + }; + self.persistent + .write() + .put_encoded(keys::peer_removed(node_id), &record, true)?; + Ok(()) + } + + /// Clear a node's removal marker: the explicit re-admission decision. + /// + /// Returns whether a marker was there to clear. The deletion this writes + /// is an ordinary tombstone -- once a node is re-admitted there is + /// nothing left that must be remembered forever. + pub fn clear_peer_removed(&self, node_id: NodeId) -> Result<bool> { + let previous = self + .persistent + .write() + .delete(keys::peer_removed(node_id))?; + Ok(previous.is_some_and(|entry| !entry.is_deleted())) + } + + /// Whether an operator has removed this node and nobody has re-admitted it. + /// + /// Fails closed on a corrupt marker: refusing sync from a node whose + /// marker cannot be read is recoverable -- the operator overwrites or + /// clears it -- while admitting a removed node's full dump can resurrect + /// every record whose delete the cluster has already collected. + pub fn is_peer_removed(&self, node_id: NodeId) -> bool { + match self + .persistent + .read() + .decode_strict::<PeerRemovalRecord>(&keys::peer_removed(node_id)) + { + Ok(record) => record.is_some(), + Err(err) => { + warn!( + "the removal marker for node {node_id} is unreadable, \ + treating the node as removed: {err:#}" + ); + true + } + } + } + + /// Watch for changes to replicated removal markers + pub fn watch_peer_removed(&self) -> watch::Receiver<()> { + self.persistent.watch_prefix(keys::PEER_REMOVED_PREFIX) + } + + /// Drop peers an operator has removed. + /// + /// The durable signal is the live `__peer_removed/{id}` marker (see + /// [`Self::mark_peer_removed`]). A tombstoned `__peer_addr/{id}` record + /// counts too: it is the only signal a removal performed by an older + /// binary leaves, and it works until the tombstone GC collects it. An + /// address that was never written does not count: bootstrap can add a + /// peer before its address record has synced in, and such a peer must + /// not be dropped for being early. + pub fn prune_removed_peers(&self) { + let peer_ids: Vec<NodeId> = self + .persistent + .read() + .status() + .peers + .iter() + .map(|peer| peer.id) + .collect(); + for peer_id in peer_ids { + // `get` filters tombstones out, so the deletion signal is only + // visible through the tombstone-inclusive accessor. + let tombstoned = self + .persistent + .read() + .get_including_tombstones(&keys::peer_addr(peer_id)) + .is_some_and(|entry| entry.is_deleted()); + if !tombstoned && !self.is_peer_removed(peer_id) { + continue; + } + warn!("dropping removed node {peer_id} from the sync peer set"); + if let Err(err) = self.remove_peer(peer_id) { + warn!("failed to remove peer {peer_id}: {err:#}"); + } + } + } + // ==================== Peer Address (in DB) ==================== /// Register a node's sync URL in DB and add to peer list for sync @@ -682,10 +1791,13 @@ impl KvStore { /// This stores the URL in KvStore (for address lookup) and also adds the node /// to the wavekv peer list (so SyncManager knows to sync with it). pub fn register_peer_url(&self, node_id: NodeId, url: &str) -> Result<()> { - // Store URL in persistent KvStore + validate_peer_url(url)?; + + // Store URL in persistent KvStore. Owned, because the value has to be a + // type a reader can name: `&str` borrows from the buffer it decodes. self.persistent .write() - .put_encoded(keys::peer_addr(node_id), &url)?; + .put_encoded(keys::peer_addr(node_id), &url.to_string(), false)?; let _ = self.add_peer(node_id); Ok(()) @@ -703,12 +1815,9 @@ impl KvStore { } pub fn update_peer_last_seen(&self, peer_id: NodeId) { - let ts = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); + let ts = now_secs(); let key = keys::last_seen_node(peer_id, self.my_node_id); - if let Err(e) = self.ephemeral.write().put_encoded(key, &ts) { + if let Err(e) = self.ephemeral.write().put_encoded(key, &ts, false) { warn!("failed to update peer {peer_id} last_seen: {e}"); } } @@ -727,16 +1836,22 @@ impl KvStore { // ==================== DNS Credential Management ==================== - /// Get a DNS credential by ID - pub fn get_dns_credential(&self, cred_id: &str) -> Option<DnsCredential> { - self.persistent.read().decode(&keys::dns_cred(cred_id)) + /// Get a DNS credential by ID. + /// + /// Fails closed on a corrupt record: silently reading it as "no such + /// credential" would make the certbot fall back to the default credential + /// and issue the domain's certificate through the wrong DNS account. + pub fn get_dns_credential(&self, cred_id: &str) -> Result<Option<DnsCredential>> { + self.persistent + .read() + .decode_strict(&keys::dns_cred(cred_id)) } /// Save a DNS credential pub fn save_dns_credential(&self, cred: &DnsCredential) -> Result<()> { self.persistent .write() - .put_encoded(keys::dns_cred(&cred.id), cred)?; + .put_encoded(keys::dns_cred(&cred.id), cred, true)?; Ok(()) } @@ -754,40 +1869,54 @@ impl KvStore { .collect() } - /// Get the default DNS credential ID - pub fn get_default_dns_credential_id(&self) -> Option<String> { - self.persistent.read().decode(keys::DNS_CRED_DEFAULT) + /// Get the default DNS credential ID. + /// + /// Fails closed on a corrupt record for the same reason as + /// [`Self::get_dns_credential`]. + pub fn get_default_dns_credential_id(&self) -> Result<Option<String>> { + self.persistent.read().decode_strict(keys::DNS_CRED_DEFAULT) } /// Set the default DNS credential ID pub fn set_default_dns_credential_id(&self, cred_id: &str) -> Result<()> { - self.persistent - .write() - .put_encoded(keys::DNS_CRED_DEFAULT.to_string(), &cred_id)?; + self.persistent.write().put_encoded( + keys::DNS_CRED_DEFAULT.to_string(), + &cred_id.to_string(), + false, + )?; Ok(()) } /// Get the default DNS credential (resolves the ID to the actual credential) - pub fn get_default_dns_credential(&self) -> Option<DnsCredential> { - let cred_id = self.get_default_dns_credential_id()?; + pub fn get_default_dns_credential(&self) -> Result<Option<DnsCredential>> { + let Some(cred_id) = self.get_default_dns_credential_id()? else { + return Ok(None); + }; self.get_dns_credential(&cred_id) } // ==================== Global Certbot Config ==================== - /// Get global certbot configuration (returns default if not set) - pub fn get_certbot_config(&self) -> GlobalCertbotConfig { - self.persistent + /// Get global certbot configuration (returns default if not set). + /// + /// Fails closed on a corrupt record: falling back to the defaults would + /// silently switch `acme_url` back to Let's Encrypt production and reset + /// every renewal interval on this node. + pub fn get_certbot_config(&self) -> Result<GlobalCertbotConfig> { + Ok(self + .persistent .read() - .decode(keys::GLOBAL_CERTBOT_CONFIG) - .unwrap_or_default() + .decode_strict(keys::GLOBAL_CERTBOT_CONFIG)? + .unwrap_or_default()) } /// Set global certbot configuration pub fn set_certbot_config(&self, config: &GlobalCertbotConfig) -> Result<()> { - self.persistent - .write() - .put_encoded(keys::GLOBAL_CERTBOT_CONFIG.to_string(), config)?; + self.persistent.write().put_encoded( + keys::GLOBAL_CERTBOT_CONFIG.to_string(), + config, + true, + )?; Ok(()) } @@ -800,11 +1929,26 @@ impl KvStore { .decode(&keys::zt_domain_config(domain)) } + /// Whether any record — readable or not — exists for the domain's config. + /// + /// [`Self::get_zt_domain_config`] cannot distinguish a missing record + /// from a corrupt one; deletion must, or a corrupt record could never be + /// removed. + pub fn zt_domain_config_exists(&self, domain: &str) -> bool { + // `get` already excludes tombstones, so Some means a live record. + self.persistent + .read() + .get(&keys::zt_domain_config(domain)) + .is_some() + } + /// Save ZT-Domain configuration pub fn save_zt_domain_config(&self, config: &ZtDomainConfig) -> Result<()> { - self.persistent - .write() - .put_encoded(keys::zt_domain_config(&config.domain), config)?; + self.persistent.write().put_encoded( + keys::zt_domain_config(&config.domain), + config, + true, + )?; Ok(()) } @@ -816,7 +1960,12 @@ impl KvStore { Ok(()) } - /// List all ZT-Domain configurations + /// List all ZT-Domain configurations. + /// + /// A record whose `domain` disagrees with the domain in its key is + /// skipped: everything downstream (certificate issuance, DNS-01 challenge, + /// `cert/{domain}/data`) is driven by the value, so honouring it would let + /// one poisoned record request a certificate for an unrelated domain. pub fn list_zt_domain_configs(&self) -> Vec<ZtDomainConfig> { let state = self.persistent.read(); state @@ -827,13 +1976,23 @@ impl KvStore { return None; } let value = entry.value.as_ref()?; - match decode(value) { - Ok(config) => Some(config), + let config: ZtDomainConfig = match decode(value) { + Ok(config) => config, Err(e) => { + crate::metrics::record_decode_failure(key); warn!("failed to decode cert config for key {key}: {e:?}"); - None + return None; } + }; + let key_domain = keys::parse_cert_domain(key)?; + if key_domain != config.domain { + warn!( + "skipping cert config at key {key}: record claims domain {}", + config.domain + ); + return None; } + Some(config) }) .collect() } @@ -883,7 +2042,7 @@ impl KvStore { pub fn save_cert_data(&self, domain: &str, data: &CertData) -> Result<()> { self.persistent .write() - .put_encoded(keys::cert_data(domain), data)?; + .put_encoded(keys::cert_data(domain), data, false)?; Ok(()) } @@ -902,6 +2061,7 @@ impl KvStore { match decode(value) { Ok(data) => Some((domain.to_string(), data)), Err(e) => { + crate::metrics::record_decode_failure(key); warn!("failed to decode cert data for key {key}: {e:?}"); None } @@ -912,29 +2072,47 @@ impl KvStore { // ==================== Global ACME Credentials ==================== - /// Get global ACME credentials (shared across all domains) - pub fn get_acme_credentials(&self) -> Option<CertCredentials> { - self.persistent.read().decode(keys::GLOBAL_ACME_CREDENTIALS) + /// Get global ACME credentials (shared across all domains). + /// + /// Fails closed on a corrupt record: a missing or deleted key is + /// `Ok(None)`, but a stored value that no longer decodes is an error. + /// Treating corruption as absence would silently register a fresh ACME + /// account that the existing account-bound CAA records refuse. + pub fn get_acme_credentials(&self) -> Result<Option<CertCredentials>> { + self.persistent + .read() + .decode_strict(keys::GLOBAL_ACME_CREDENTIALS) + .context("corrupt ACME credentials record in KvStore") } /// Save global ACME credentials pub fn save_acme_credentials(&self, creds: &CertCredentials) -> Result<()> { - self.persistent - .write() - .put_encoded(keys::GLOBAL_ACME_CREDENTIALS.to_string(), creds)?; + self.persistent.write().put_encoded( + keys::GLOBAL_ACME_CREDENTIALS.to_string(), + creds, + false, + )?; Ok(()) } - /// Get global ACME attestation (TDX quote of account URI) - pub fn get_acme_attestation(&self) -> Option<AcmeAttestation> { - self.persistent.read().decode(keys::GLOBAL_ACME_ATTESTATION) + /// Get global ACME attestation (TDX quote of account URI). + /// + /// Fails closed on a corrupt record: reporting "no attestation" for an + /// account that does have one lets a verifier conclude the ACME account is + /// unattested. + pub fn get_acme_attestation(&self) -> Result<Option<AcmeAttestation>> { + self.persistent + .read() + .decode_strict(keys::GLOBAL_ACME_ATTESTATION) } /// Save global ACME attestation pub fn save_acme_attestation(&self, attestation: &AcmeAttestation) -> Result<()> { - self.persistent - .write() - .put_encoded(keys::GLOBAL_ACME_ATTESTATION.to_string(), attestation)?; + self.persistent.write().put_encoded( + keys::GLOBAL_ACME_ATTESTATION.to_string(), + attestation, + false, + )?; Ok(()) } @@ -948,14 +2126,11 @@ impl KvStore { /// Try to acquire certificate renew lock /// Returns true if lock acquired, false if already locked by another node pub fn try_acquire_cert_lock(&self, domain: &str, lock_timeout_secs: u64) -> bool { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); + let now = now_secs(); if let Some(existing) = self.get_cert_lock(domain) { // Check if lock is still valid (not expired) - if now < existing.started_at + lock_timeout_secs { + if now < existing.started_at.saturating_add(lock_timeout_secs) { return false; } } @@ -967,7 +2142,7 @@ impl KvStore { }; self.persistent .write() - .put_encoded(keys::cert_lock(domain), &lock) + .put_encoded(keys::cert_lock(domain), &lock, false) .is_ok() } @@ -977,6 +2152,75 @@ impl KvStore { Ok(()) } + /// Try to acquire the lock over the cluster's shared ACME account. + /// + /// Every operation over that account takes it -- rotation, CAA + /// reconciliation, and first-use registration -- so they are ordered + /// against each other across nodes as well as within one; see + /// [`crate::distributed_certbot::DistributedCertBot`]. + /// + /// Returns the lock value that was written; pass it back to + /// [`Self::release_rotation_lock`] so a rotation that outlived the timeout + /// cannot delete the lock of the node that took over. + /// + /// Best-effort only: WaveKV is last-writer-wins without compare-and-swap, + /// so two nodes can both acquire during a replication gap. This narrows the + /// window for concurrent rotation from the full rotation duration to the + /// replication latency; it is not mutual exclusion. A crashed holder is + /// covered by the timeout. + pub fn try_acquire_rotation_lock(&self, lock_timeout_secs: u64) -> Option<CertRenewLock> { + let now = now_secs(); + + if let Some(existing) = self.get_rotation_lock() { + // Check if lock is still valid (not expired) + if now < existing.started_at.saturating_add(lock_timeout_secs) { + return None; + } + } + + let lock = CertRenewLock { + started_at: now, + started_by: self.my_node_id, + }; + self.persistent + .write() + .put_encoded(keys::GLOBAL_ACME_ROTATION_LOCK.to_string(), &lock, false) + .ok()?; + Some(lock) + } + + /// Get the global ACME credential rotation lock + pub fn get_rotation_lock(&self) -> Option<CertRenewLock> { + self.persistent + .read() + .decode(keys::GLOBAL_ACME_ROTATION_LOCK) + } + + /// Release the global ACME credential rotation lock. + /// + /// Only deletes the lock when the currently visible value is the one that + /// `acquired` wrote: a rotation that outlived the lock timeout must not + /// delete the lock of the node that took over (which would let a third + /// rotation start concurrently). Like acquisition, the check is + /// best-effort under WaveKV's last-writer-wins replication. + pub fn release_rotation_lock(&self, acquired: &CertRenewLock) -> Result<()> { + if let Some(current) = self.get_rotation_lock() { + if current.started_by != acquired.started_by + || current.started_at != acquired.started_at + { + warn!( + "not releasing ACME rotation lock: node {} took it over after this rotation exceeded the lock timeout", + current.started_by + ); + return Ok(()); + } + } + self.persistent + .write() + .delete(keys::GLOBAL_ACME_ROTATION_LOCK.to_string())?; + Ok(()) + } + // ==================== Certificate Attestation ==================== /// Get the latest attestation for a domain @@ -993,9 +2237,10 @@ impl KvStore { state.put_encoded( keys::cert_attestation_history(domain, attestation.generated_at), attestation, + false, )?; // Update latest - state.put_encoded(keys::cert_attestation_latest(domain), attestation)?; + state.put_encoded(keys::cert_attestation_latest(domain), attestation, false)?; Ok(()) } @@ -1015,6 +2260,7 @@ impl KvStore { match decode(value) { Ok(att) => Some(att), Err(e) => { + crate::metrics::record_decode_failure(key); warn!("failed to decode attestation for key {key}: {e:?}"); None } @@ -1033,3 +2279,1754 @@ impl KvStore { self.persistent.watch_prefix(keys::CERT_PREFIX) } } + +/// Move an unreadable WaveKV data dir aside, returning the new path. +/// +/// Renaming keeps the bytes for post-mortem analysis and guarantees the +/// gateway never starts on half-readable state. +fn quarantine_data_dir(data_dir: &Path) -> Result<std::path::PathBuf> { + anyhow::ensure!( + data_dir.exists(), + "WaveKV data dir {} does not exist", + data_dir.display() + ); + let stamp = now_secs(); + for attempt in 0..u32::MAX { + let suffix = if attempt == 0 { + format!("corrupt.{stamp}") + } else { + format!("corrupt.{stamp}.{attempt}") + }; + let mut target = data_dir.as_os_str().to_owned(); + target.push("."); + target.push(&suffix); + let target = std::path::PathBuf::from(target); + if target.exists() { + continue; + } + std::fs::rename(data_dir, &target) + .with_context(|| format!("failed to rename {}", data_dir.display()))?; + return Ok(target); + } + anyhow::bail!("no free quarantine path for {}", data_dir.display()) +} + +fn validate_peer_url(url: &str) -> Result<()> { + let parsed = reqwest::Url::parse(url).context("invalid peer URL")?; + anyhow::ensure!( + matches!(parsed.scheme(), "http" | "https"), + "peer URL scheme must be http or https" + ); + anyhow::ensure!(parsed.host_str().is_some(), "peer URL must include a host"); + anyhow::ensure!( + parsed.username().is_empty() && parsed.password().is_none(), + "peer URL must not contain credentials" + ); + Ok(()) +} + +#[cfg(test)] +mod acme_credentials_tests { + use super::*; + + fn test_kv(data_dir: &std::path::Path) -> KvStore { + KvStore::new(1, vec![], data_dir, None).expect("failed to create kv store") + } + + #[test] + fn missing_and_deleted_credentials_are_absent_not_errors() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let kv = test_kv(dir.path()); + assert!(kv + .get_acme_credentials() + .expect("missing key should not error") + .is_none()); + + kv.save_acme_credentials(&CertCredentials { + acme_credentials: "{}".to_string(), + }) + .expect("save should succeed"); + kv.persistent + .write() + .delete(keys::GLOBAL_ACME_CREDENTIALS.to_string()) + .expect("delete should succeed"); + assert!(kv + .get_acme_credentials() + .expect("tombstone should not error") + .is_none()); + } + + #[test] + fn corrupt_credentials_record_fails_closed() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let kv = test_kv(dir.path()); + kv.persistent + .write() + .put( + keys::GLOBAL_ACME_CREDENTIALS.to_string(), + b"not-messagepack".to_vec(), + ) + .expect("raw put should succeed"); + let err = kv + .get_acme_credentials() + .expect_err("corrupt record must not read as absent"); + assert!( + err.to_string().contains("corrupt ACME credentials"), + "unexpected error: {err}" + ); + } + + #[test] + fn lease_expiry_does_not_overflow() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let kv = test_kv(dir.path()); + kv.persistent + .write() + .put_encoded( + keys::GLOBAL_ACME_ROTATION_LOCK.to_string(), + &CertRenewLock { + started_at: u64::MAX, + started_by: 2, + }, + false, + ) + .expect("lock write should succeed"); + + assert!( + kv.try_acquire_rotation_lock(600).is_none(), + "a non-expired lock with a saturated expiry must remain held" + ); + + kv.persistent + .write() + .put_encoded( + keys::cert_lock("overflow.example"), + &CertRenewLock { + started_at: u64::MAX, + started_by: 2, + }, + false, + ) + .expect("certificate lock write should succeed"); + assert!( + !kv.try_acquire_cert_lock("overflow.example", 600), + "a non-expired certificate lock with a saturated expiry must remain held" + ); + } +} + +/// KV values replicate between gateways, so their MessagePack encoding is a wire +/// contract across a mixed-version cluster and across a node's own restart. Named +/// maps keep that contract on field names rather than field order, so a value type +/// can gain a field without breaking gateways still running an older build. +#[cfg(test)] +mod value_encoding_tests { + use super::*; + use serde::{Deserialize, Serialize}; + + /// A value type that has since gained a field. Stands in for an older gateway + /// reading a record written by this build. + #[derive(Debug, Serialize, Deserialize)] + struct ReducedCertRenewLock { + started_at: u64, + } + + /// True when `bytes` opens with a MessagePack map header of any width. The header + /// widens from fixmap to map16 at 16 entries, so matching on the fixmap range alone + /// would start failing precisely when a value type grows past 15 fields — the case + /// this encoding exists to support. + fn starts_with_msgpack_map(bytes: &[u8]) -> bool { + matches!(bytes.first().copied(), Some(0x80..=0x8f | 0xde | 0xdf)) + } + + /// The positional counterpart: fixarray, array16, or array32. + fn starts_with_msgpack_array(bytes: &[u8]) -> bool { + matches!(bytes.first().copied(), Some(0x90..=0x9f | 0xdc | 0xdd)) + } + + #[test] + fn values_are_encoded_as_named_maps() { + let encoded = encode(&CertRenewLock { + started_at: 1_700_000_000, + started_by: 7, + }) + .expect("encode should succeed"); + + assert!( + starts_with_msgpack_map(&encoded), + "values must encode as MessagePack maps, not positional arrays" + ); + } + + /// New writer, old reader: a peer whose struct predates a field must skip it. + #[test] + fn named_values_decode_against_a_reduced_field_set() { + let encoded = encode(&CertRenewLock { + started_at: 1_700_000_000, + started_by: 7, + }) + .expect("encode should succeed"); + + let reduced: ReducedCertRenewLock = + decode(&encoded).expect("a reader without started_by must skip the field, not fail"); + assert_eq!(reduced.started_at, 1_700_000_000); + } + + /// Old writer, new reader: records written before this change are positional + /// arrays and must keep decoding after an upgrade. + #[test] + fn legacy_positional_records_are_still_readable() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let kv = KvStore::new(1, vec![], dir.path(), None).expect("failed to create kv store"); + + let legacy = rmp_serde::encode::to_vec(&CertRenewLock { + started_at: 1_700_000_000, + started_by: 7, + }) + .expect("legacy encode should succeed"); + assert!( + starts_with_msgpack_array(&legacy), + "fixture must be a positional array to exercise the legacy path" + ); + + kv.persistent + .write() + .put(keys::cert_lock("legacy.example"), legacy) + .expect("put should succeed"); + + let decoded = kv + .get_cert_lock("legacy.example") + .expect("a record written by an older gateway must stay readable"); + assert_eq!(decoded.started_at, 1_700_000_000); + assert_eq!(decoded.started_by, 7); + } + + /// `DnsCredential` is the most demanding value type in the set: it nests an + /// internally tagged enum and a field with a custom `serde(with)` codec, both of + /// which behave differently across self-describing and positional encodings. + #[test] + fn nested_tagged_enums_and_custom_codecs_survive_both_encodings() { + let credential = DnsCredential { + id: "cred-1".to_string(), + name: "primary".to_string(), + provider: DnsProvider::Cloudflare { + api_token: "token".to_string(), + api_url: Some("https://api.cloudflare.com/client/v4".to_string()), + }, + max_dns_wait: Duration::from_secs(90), + dns_txt_ttl: 60, + created_at: 1_700_000_000, + updated_at: 1_700_000_001, + }; + + for (label, encoded) in [ + ("named", encode(&credential).expect("named encode")), + ( + "legacy positional", + rmp_serde::encode::to_vec(&credential).expect("positional encode"), + ), + ] { + let decoded: DnsCredential = + decode(&encoded).unwrap_or_else(|err| panic!("{label} decode failed: {err}")); + assert_eq!(decoded.id, credential.id, "{label}"); + assert_eq!(decoded.max_dns_wait, credential.max_dns_wait, "{label}"); + assert_eq!(decoded.dns_txt_ttl, credential.dns_txt_ttl, "{label}"); + let DnsProvider::Cloudflare { api_token, api_url } = decoded.provider; + assert_eq!(api_token, "token", "{label}"); + assert_eq!( + api_url.as_deref(), + Some("https://api.cloudflare.com/client/v4"), + "{label}" + ); + } + } + + /// A ZT domain stored before `dns-persist-01` existed has no `challenge` + /// field, and must keep decoding as the method it was actually using. + #[test] + fn a_zt_domain_without_a_challenge_field_decodes_as_dns01() { + /// The shape `ZtDomainConfig` had before the field was added. + #[derive(Serialize)] + struct LegacyZtDomainConfig { + domain: String, + dns_cred_id: Option<String>, + port: u16, + node: Option<u32>, + priority: i32, + } + + let legacy = LegacyZtDomainConfig { + domain: "app.example.com".to_string(), + dns_cred_id: Some("cred-1".to_string()), + port: 443, + node: None, + priority: 7, + }; + + for (label, encoded) in [ + ("named", encode(&legacy).expect("named encode")), + ( + "legacy positional", + rmp_serde::encode::to_vec(&legacy).expect("positional encode"), + ), + ] { + let decoded: ZtDomainConfig = + decode(&encoded).unwrap_or_else(|err| panic!("{label} decode failed: {err}")); + assert_eq!(decoded.domain, "app.example.com", "{label}"); + assert_eq!(decoded.priority, 7, "{label}"); + assert_eq!(decoded.challenge, ChallengeKind::Dns01, "{label}"); + } + } + + /// The configured window has to reach the store, not just the config file. + /// + /// An fsync per write runs under the store lock, so it bounds how fast this + /// gateway accepts registrations; the window is what buys that back, and it + /// buys nothing if the interval stops at `SyncConfig`. + /// + /// The window here is long enough that no scheduling delay can end it + /// mid-test: the property is "not yet due", and a stalled runner must not be + /// able to turn that into a failure. + #[test] + fn a_configured_window_holds_writes_out_of_the_disk() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let kv = KvStore::new(1, vec![], dir.path(), Some(Duration::from_secs(3_600))) + .expect("kv store"); + + kv.set_node_status(1, NodeStatus::Up).expect("write"); + + assert_eq!( + kv.persistent().read().wal_sync_count(), + 0, + "a write inside the window must not reach the disk" + ); + assert!( + !kv.sync_wal_if_due().expect("sync check"), + "nothing is due before the window elapses" + ); + } + + /// And the window has to end. The only timing this depends on is sleeping + /// for longer than the window, which is safe in the direction that matters. + #[test] + fn an_elapsed_window_forces_the_log() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let window = Duration::from_millis(20); + let kv = KvStore::new(1, vec![], dir.path(), Some(window)).expect("kv store"); + + kv.set_node_status(1, NodeStatus::Up).expect("write"); + std::thread::sleep(window * 5); + + assert!(kv.sync_wal_if_due().expect("sync"), "the window elapsed"); + assert_eq!(kv.persistent().read().wal_sync_count(), 1); + assert!( + !kv.sync_wal_if_due().expect("sync check"), + "a second call with nothing written must not force the disk again" + ); + } + + /// Without a window every write is on the disk before it returns, which is + /// what every release before this one did and what a single-node gateway + /// holding the only copy of its ACME account still wants. + #[test] + fn without_a_window_every_write_reaches_the_disk_before_it_returns() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let kv = KvStore::new(1, vec![], dir.path(), None).expect("kv store"); + + kv.set_node_status(1, NodeStatus::Up).expect("write"); + kv.set_node_status(2, NodeStatus::Down).expect("write"); + + assert_eq!(kv.persistent().read().wal_sync_count(), 2); + assert!( + !kv.sync_wal_if_due().expect("sync check"), + "with no window there is never anything owing" + ); + } + + /// An instance record as some later release will declare it. + #[derive(Debug, Serialize, Deserialize)] + struct FutureInstanceRecord { + app_id: String, + ip: Ipv4Addr, + public_key: String, + reg_time: u64, + /// The field this build has never heard of. + health_probe_path: String, + } + + /// The write-path half of the mixed-version contract. Skipping an unknown + /// field on read is not enough: this build re-encodes the record from the + /// fields it declares, so without the merge a single re-registration by an + /// older node erases a newer node's field for the entire cluster. + #[test] + fn a_record_keeps_the_fields_its_writer_does_not_know() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let kv = KvStore::new(1, vec![], dir.path(), None).expect("failed to create kv store"); + + let written_by_a_newer_node = encode(&FutureInstanceRecord { + app_id: "app".to_string(), + ip: Ipv4Addr::new(10, 0, 0, 1), + public_key: "pubkey".to_string(), + reg_time: 1_700_000_000, + health_probe_path: "/healthz".to_string(), + }) + .expect("encode should succeed"); + kv.persistent + .write() + .put(keys::inst("app"), written_by_a_newer_node) + .expect("put should succeed"); + + // This build re-registers the instance, knowing nothing of the new field. + kv.sync_instance( + "app", + &InstanceRecord { + app_id: "app".to_string(), + ip: Ipv4Addr::new(10, 0, 0, 2), + public_key: "pubkey".to_string(), + reg_time: 1_700_000_100, + port_policy: None, + port_policy_hash: String::new(), + health_check: None, + }, + ) + .expect("sync should succeed"); + + let stored = kv + .persistent + .read() + .get(&keys::inst("app")) + .and_then(|entry| entry.value) + .expect("record should exist"); + let seen_by_a_newer_node: FutureInstanceRecord = + decode(&stored).expect("the newer node must still read its own field"); + assert_eq!( + seen_by_a_newer_node.health_probe_path, "/healthz", + "the unknown field must survive a write by a build that cannot see it" + ); + assert_eq!( + seen_by_a_newer_node.ip, + Ipv4Addr::new(10, 0, 0, 2), + "the writer's own fields must still take effect" + ); + let seen_by_this_build: InstanceRecord = + decode(&stored).expect("this build must still read the record"); + assert_eq!(seen_by_this_build.reg_time, 1_700_000_100); + } + + /// A certificate is a complete new fact on every write, so nothing may be + /// carried across one. Attributing a previous certificate's field to the one + /// just issued is worse than dropping the field: absent is a case the newer + /// reader already handles, stale is not. + #[test] + fn a_snapshot_does_not_inherit_the_previous_writes_fields() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let kv = KvStore::new(1, vec![], dir.path(), None).expect("failed to create kv store"); + + #[derive(Debug, Serialize, Deserialize)] + struct FutureCertData { + cert_pem: String, + key_pem: String, + not_after: u64, + issued_by: NodeId, + issued_at: u64, + chain_pem: String, + } + + kv.persistent + .write() + .put( + keys::cert_data("a.example"), + encode(&FutureCertData { + cert_pem: "old-cert".to_string(), + key_pem: "old-key".to_string(), + not_after: 1_700_000_000, + issued_by: 2, + issued_at: 1_600_000_000, + chain_pem: "old-chain".to_string(), + }) + .expect("encode should succeed"), + ) + .expect("put should succeed"); + + kv.save_cert_data( + "a.example", + &CertData { + cert_pem: "new-cert".to_string(), + key_pem: "new-key".to_string(), + not_after: 1_800_000_000, + issued_by: 1, + issued_at: 1_700_000_100, + }, + ) + .expect("save should succeed"); + + let stored = kv + .persistent + .read() + .get(&keys::cert_data("a.example")) + .and_then(|entry| entry.value) + .expect("record should exist"); + assert!( + decode::<FutureCertData>(&stored).is_err(), + "the previous certificate's chain must not be attached to the new one" + ); + } +} + +/// Gateway-layer tests for the WaveKV sync wire and admission policy. +#[cfg(test)] +mod sync_wire_tests { + use super::*; + use wavekv::sync::SyncEnvelope; + + fn store(dir: &std::path::Path, id: NodeId, peers: Vec<NodeId>) -> KvStore { + KvStore::new(id, peers, dir, None).expect("failed to create kv store") + } + + #[test] + fn a_sync_envelope_survives_the_transport_framing() { + use flate2::{read::GzDecoder, write::GzEncoder, Compression}; + use std::io::{Read, Write}; + + let dir = tempfile::tempdir().expect("tempdir"); + let kv = store(dir.path(), 1, vec![2]); + kv.persistent() + .write() + .put(keys::peer_addr(1), b"https://a.example".to_vec()) + .expect("put"); + + // Requests deliberately carry no digest: sending it would let any responder + // echo it back and forge agreement forever. So frame a *response*, which is + // the direction the digest actually travels. + assert!(kv + .persistent() + .read() + .prepare_sync(2, Vec::new()) + .digest + .is_none()); + let env = kv + .persistent() + .write() + .handle_envelope(SyncEnvelope::new(2, Vec::new()), Vec::new()) + .expect("respond"); + assert!(!env.entries.is_empty()); + + let mut encoder = GzEncoder::new(Vec::new(), Compression::fast()); + encoder.write_all(&env.encode().expect("encode")).unwrap(); + let wire = encoder.finish().unwrap(); + + let mut plain = Vec::new(); + GzDecoder::new(&wire[..]).read_to_end(&mut plain).unwrap(); + let decoded = SyncEnvelope::decode(&plain).expect("decode"); + + assert_eq!(decoded.sender_id, 1); + assert_eq!(decoded.entries.len(), env.entries.len()); + assert!( + decoded.digest.is_some(), + "the digest drives divergence detection" + ); + } +} + +/// A production WaveKV 1.0 gateway is upgraded in place while stopped. There is no +/// mixed-version cluster protocol to preserve, but its persistent snapshot and WAL are +/// an on-disk compatibility contract. +#[cfg(test)] +mod wavekv_v1_migration_tests { + use super::*; + + #[test] + fn an_upgraded_gateway_opens_and_preserves_a_v1_data_directory() { + let dir = tempfile::tempdir().expect("tempdir"); + let key = keys::peer_addr(7); + let value = b"https://gateway-7.example:8011".to_vec(); + let wal_key = keys::peer_addr(8); + let wal_value = b"https://gateway-8.example:8011".to_vec(); + + { + let v1 = wavekv_v1::Node::new_with_persistence(1, Vec::new(), dir.path()) + .expect("create v1 store"); + v1.write() + .put(key.clone(), value.clone()) + .expect("write v1 data"); + v1.persist_if_dirty().expect("persist v1 snapshot"); + v1.write() + .put(wal_key.clone(), wal_value.clone()) + .expect("write trailing v1 WAL entry"); + } + + let upgraded = + KvStore::new(1, Vec::new(), dir.path(), None).expect("open v1 data after upgrade"); + assert_eq!( + upgraded + .persistent() + .read() + .get(&key) + .and_then(|entry| entry.value), + Some(value.clone()), + "the stopped single-node upgrade must preserve the replicated state" + ); + assert_eq!( + upgraded + .persistent() + .read() + .get(&wal_key) + .and_then(|entry| entry.value), + Some(wal_value.clone()), + "the upgrade must replay v1 WAL entries written after the snapshot" + ); + + let new_key = keys::peer_addr(9); + let new_value = b"https://gateway-9.example:8011".to_vec(); + upgraded + .persistent() + .write() + .put(new_key.clone(), new_value.clone()) + .expect("write data after upgrade"); + upgraded.persist_if_dirty().expect("persist upgraded data"); + drop(upgraded); + + let restarted = + KvStore::new(1, Vec::new(), dir.path(), None).expect("restart upgraded store"); + for (key, expected) in [(key, value), (wal_key, wal_value), (new_key, new_value)] { + assert_eq!( + restarted + .persistent() + .read() + .get(&key) + .and_then(|entry| entry.value), + Some(expected), + "all migrated and post-upgrade data must survive an upgraded restart: {key}" + ); + } + } +} + +#[cfg(test)] +mod decompression_tests { + use super::*; + use std::io::Write; + + fn gzip(bytes: &[u8]) -> Vec<u8> { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(bytes).expect("write"); + encoder.finish().expect("finish") + } + + /// A bomb rejected by size, not by decoding: gzip expands by three orders of + /// magnitude on attacker-chosen input, so the cap on the compressed body + /// bounds nothing on its own. + #[test] + fn an_expansion_past_the_limit_is_refused() { + let bomb = gzip(&vec![0u8; 512 * 1024]); + assert!(gunzip_bounded(&bomb, 4096).is_err()); + assert!(bomb.len() < 4096, "the fixture must be small compressed"); + } + + /// The limit is inclusive, so a payload landing exactly on it still decodes. + /// Without this the bound could tighten by a byte and only the bomb test would + /// still pass. + #[test] + fn a_payload_exactly_on_the_limit_still_decompresses() { + let exact = gzip(&vec![7u8; 4096]); + let out = gunzip_bounded(&exact, 4096).expect("must be accepted"); + assert_eq!(out.len(), 4096); + assert!(gunzip_bounded(&gzip(&vec![7u8; 4097]), 4096).is_err()); + } +} + +#[cfg(test)] +mod corruption_tests { + use super::*; + + pub(super) fn test_kv(data_dir: &std::path::Path) -> KvStore { + KvStore::new(1, vec![], data_dir, None).expect("failed to create kv store") + } + + pub(super) fn put_raw(kv: &KvStore, key: &str, value: &[u8]) { + kv.persistent + .write() + .put(key.to_string(), value.to_vec()) + .expect("raw put should succeed"); + } + + fn seed_instance(kv: &KvStore, id: &str) { + kv.sync_instance( + id, + &InstanceRecord { + app_id: "app".to_string(), + ip: "10.0.0.20".parse().unwrap(), + public_key: "key".to_string(), + reg_time: 1, + port_policy: None, + port_policy_hash: String::new(), + health_check: None, + }, + ) + .expect("seed"); + kv.sync_connections(id, 0).expect("seed conn"); + kv.sync_instance_handshake(id, 1).expect("seed handshake"); + } + + /// A telemetry failure must neither abandon the remaining keys nor report + /// that durable instance removal failed. + #[test] + fn a_failed_telemetry_delete_does_not_fail_or_abandon_the_remaining_keys() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = test_kv(dir.path()); + seed_instance(&kv, "cvm"); + + kv.fail_writes_for_test(&[InstanceKey::Connections]); + assert!( + kv.sync_delete_instance("cvm") + .expect("a telemetry failure must not fail removal"), + "the live instance record was removed" + ); + kv.fail_writes_for_test(&[]); + + assert!( + kv.persistent + .read() + .get(&keys::inst("cvm")) + .is_none_or(|entry| entry.is_deleted()), + "the inst/ tombstone should still have been written" + ); + assert!( + kv.ephemeral + .read() + .get(&keys::handshake("cvm", kv.my_node_id)) + .is_none_or(|entry| entry.is_deleted()), + "the handshake delete must be attempted even after conn/ failed" + ); + } + + /// An orphaned `admin/` record is persistent -- a recycled instance id + /// would inherit it -- but it does not change whether the CVM was removed, + /// so the failure is logged, the remaining keys are still deleted, and the + /// removal reports the fate of the `inst/` tombstone alone. + #[test] + fn a_failed_override_delete_does_not_fail_the_removal() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = test_kv(dir.path()); + seed_instance(&kv, "cvm"); + kv.set_instance_gate("cvm", false).expect("seed gate"); + + kv.fail_writes_for_test(&[InstanceKey::Gate]); + assert!( + kv.sync_delete_instance("cvm") + .expect("an override failure must not fail removal"), + "the live instance record was removed" + ); + kv.fail_writes_for_test(&[]); + + assert!( + kv.persistent + .read() + .get(&keys::inst("cvm")) + .is_none_or(|entry| entry.is_deleted()), + "the inst/ tombstone should still have been written" + ); + assert!( + kv.persistent + .read() + .get(&keys::admin_ready("cvm")) + .is_some_and(|entry| !entry.is_deleted()), + "the failed gate delete leaves the orphan this test is about" + ); + assert!( + kv.ephemeral + .read() + .get(&keys::conn("cvm", kv.my_node_id)) + .is_none_or(|entry| entry.is_deleted()), + "the conn/ delete must be attempted even after the gate failed" + ); + } + + /// Only the `inst/` failure reaches the caller -- the other deletes are + /// logged, never returned -- and its message has to name the record, so + /// the operator learns the removal itself is what did not take. + #[test] + fn the_persistent_failure_is_the_one_reported() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = test_kv(dir.path()); + seed_instance(&kv, "cvm"); + + kv.fail_writes_for_test(&[ + InstanceKey::Instance, + InstanceKey::Connections, + InstanceKey::Handshake, + ]); + let err = kv + .sync_delete_instance("cvm") + .expect_err("all three were made to fail"); + kv.fail_writes_for_test(&[]); + + let message = format!("{err:#}"); + assert!( + message.contains(InstanceKey::Instance.name()), + "expected the instance record to be named, got: {message}" + ); + assert!( + !message.contains(InstanceKey::Handshake.name()), + "the ephemeral failure must not mask it, got: {message}" + ); + } + + #[test] + fn a_clean_delete_still_reports_that_the_record_existed() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = test_kv(dir.path()); + seed_instance(&kv, "cvm"); + assert!(kv.sync_delete_instance("cvm").expect("delete")); + assert!( + !kv.sync_delete_instance("cvm").expect("second delete"), + "a tombstone is not a live record" + ); + } + + /// Write an instance record the way the build before the overrides moved + /// out of it did: with both operator fields inside. + fn seed_legacy_record( + kv: &KvStore, + id: &str, + port_policy: Option<PortPolicy>, + ready: Option<bool>, + ) { + #[derive(serde::Serialize)] + struct LegacyRecord { + app_id: String, + ip: std::net::Ipv4Addr, + public_key: String, + reg_time: u64, + port_policy: Option<PortPolicy>, + port_policy_hash: String, + admin_port_policy: Option<PortPolicy>, + ready: Option<bool>, + } + let encoded = encode(&LegacyRecord { + app_id: "app".to_string(), + ip: "10.0.0.20".parse().unwrap(), + public_key: format!("key-{id}"), + reg_time: 1, + port_policy: None, + port_policy_hash: String::new(), + admin_port_policy: port_policy, + ready, + }) + .expect("encode"); + put_raw(kv, &keys::inst(id), &encoded); + } + + fn restrictive() -> PortPolicy { + PortPolicy { + ports: BTreeMap::from([(8443, PortFlags { pp: false })]), + restrict_mode: true, + } + } + + /// A gateway upgrading from a build that kept the overrides inside the + /// instance record has them only there, and nothing rewrites that record on + /// the operator's behalf. Losing the port-policy override silently widens + /// the ports the app serves, so the move has to happen on load. + #[test] + fn overrides_left_in_an_instance_record_are_moved_to_their_own_keys() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = test_kv(dir.path()); + seed_legacy_record(&kv, "legacy", Some(restrictive()), Some(false)); + + // Until the move, the read-side fallback is what keeps them in force. + assert_eq!( + kv.legacy_instance_overrides()["legacy"], + LegacyOverrides { + admin_port_policy: Some(restrictive()), + ready: Some(false), + } + ); + assert_eq!( + kv.migrate_legacy_instance_overrides(), + vec![ + keys::admin_ready("legacy"), + keys::admin_port_policy("legacy") + ] + ); + assert_eq!(kv.instance_gate("legacy").unwrap(), Some(false)); + assert_eq!( + kv.instance_port_policy_override("legacy").unwrap(), + Some(PortPolicyOverride::Set(restrictive())) + ); + assert!( + kv.migrate_legacy_instance_overrides().is_empty(), + "the move is not repeated once the keys exist" + ); + } + + /// One key per override, so a legacy record holding only one of them moves + /// only that one -- and the other stays open to being set later. + #[test] + fn each_override_is_moved_on_its_own() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = test_kv(dir.path()); + seed_legacy_record(&kv, "legacy", None, Some(false)); + assert_eq!( + kv.migrate_legacy_instance_overrides(), + vec![keys::admin_ready("legacy")] + ); + assert_eq!(kv.instance_gate("legacy").unwrap(), Some(false)); + assert_eq!( + kv.instance_port_policy_override("legacy").unwrap(), + None, + "the port-policy key was never written" + ); + } + + /// Two overrides in one record meant setting one on this node discarded a + /// peer's unsynced change to the other, because WaveKV resolves a conflict + /// by taking a whole value. Separate keys are what make them independent. + #[test] + fn setting_one_override_leaves_the_other_alone() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = test_kv(dir.path()); + // What a peer set, arriving through sync. + kv.set_instance_port_policy_override("cvm", Some(restrictive())) + .expect("peer override"); + // What this node sets, from a snapshot that never saw it. + kv.set_instance_gate("cvm", false).expect("gate"); + + assert_eq!(kv.instance_gate("cvm").unwrap(), Some(false)); + assert_eq!( + kv.instance_port_policy_override("cvm").unwrap(), + Some(PortPolicyOverride::Set(restrictive())) + ); + } + + /// The instance record keeps its copy through the upgrade -- this build no + /// longer declares those fields, so `carry_unknown_fields` preserves them + /// and a node still on the previous build finds them where it left them. + /// That is also why an empty override record has to stop the move: without + /// it, clearing an override here would be undone by the stale copy. + #[test] + fn a_cleared_override_is_not_resurrected_from_the_instance_record() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = test_kv(dir.path()); + seed_legacy_record(&kv, "legacy", Some(restrictive()), None); + assert_eq!( + kv.migrate_legacy_instance_overrides(), + vec![keys::admin_port_policy("legacy")] + ); + + // The operator clears it. Cleared is written, not deleted. + kv.set_instance_port_policy_override("legacy", None) + .expect("clear"); + assert_eq!( + kv.instance_port_policy_override("legacy").unwrap(), + Some(PortPolicyOverride::Cleared) + ); + + assert!( + kv.migrate_legacy_instance_overrides().is_empty(), + "a key that says \"cleared\" is still an answer" + ); + assert_eq!( + kv.instance_port_policy_override("legacy").unwrap(), + Some(PortPolicyOverride::Cleared), + "the copy in the instance record must not come back" + ); + } + + /// Instance ids are recycled. An id reused after a delete would inherit the + /// gate and port policy an operator set for whatever ran under it before. + #[test] + fn deleting_an_instance_tombstones_its_overrides() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = test_kv(dir.path()); + seed_instance(&kv, "cvm"); + kv.set_instance_gate("cvm", false).expect("seed gate"); + kv.set_instance_port_policy_override("cvm", Some(restrictive())) + .expect("seed override"); + + kv.sync_delete_instance("cvm").expect("delete"); + assert!( + kv.instance_gate("cvm").unwrap().is_none() + && kv.instance_port_policy_override("cvm").unwrap().is_none(), + "the override record must not outlive the instance" + ); + } + + /// A build that adds a field must not make the record unreadable to one + /// that does not know it: msgpack named maps let the older reader skip what + /// it does not recognise. If this ever fails, widening `InstanceRecord` has + /// stopped being a no-op for older nodes. + /// + /// `admin_port_policy` and `ready` stand in for the retired case as well as + /// the future one: this build no longer declares them, so they arrive here + /// the same way a field from a newer peer would. + #[test] + fn an_instance_record_with_an_unknown_field_still_decodes() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let kv = test_kv(dir.path()); + + #[derive(serde::Serialize)] + struct FutureRecord<'a> { + app_id: &'a str, + ip: std::net::Ipv4Addr, + public_key: &'a str, + reg_time: u64, + port_policy: Option<PortPolicy>, + port_policy_hash: &'a str, + admin_port_policy: Option<PortPolicy>, + ready: Option<bool>, + reason: &'a str, + } + let widened = encode(&FutureRecord { + app_id: "app", + ip: "10.0.0.5".parse().unwrap(), + public_key: "k", + reg_time: 1, + port_policy: None, + port_policy_hash: "", + admin_port_policy: None, + ready: Some(false), + reason: "under investigation", + }) + .expect("encode should succeed"); + put_raw(&kv, &keys::inst("future"), &widened); + + let loaded = kv.load_all_instances(); + assert!(loaded.undecodable.is_empty(), "{:?}", loaded.undecodable); + assert_eq!(loaded.decoded["future"].public_key, "k"); + } + + #[test] + fn a_corrupt_certbot_config_does_not_read_as_the_default() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let kv = test_kv(dir.path()); + // Absent means "use the defaults" — that part must keep working. + let default = kv + .get_certbot_config() + .expect("missing key should not error"); + assert!(default.acme_url.is_empty()); + + kv.set_certbot_config(&GlobalCertbotConfig { + acme_url: "https://acme-staging.example/directory".to_string(), + ..Default::default() + }) + .expect("save should succeed"); + put_raw(&kv, keys::GLOBAL_CERTBOT_CONFIG, b"not-messagepack"); + // Reading the corrupt record as the default would silently move + // issuance back to Let's Encrypt production. + assert!(kv.get_certbot_config().is_err()); + } + + #[test] + fn a_corrupt_certbot_config_can_still_be_replaced_by_an_operator() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let kv = test_kv(dir.path()); + put_raw(&kv, keys::GLOBAL_CERTBOT_CONFIG, b"not-messagepack"); + + // The key is a singleton with no delete RPC, so overwriting it is the + // only repair path there is; the write must not inherit the read's + // fail-closed behaviour. (Which fields an operator has to supply to be + // allowed to overwrite is decided one layer up, in `admin_service`.) + kv.set_certbot_config(&GlobalCertbotConfig { + acme_url: "https://acme-staging.example/directory".to_string(), + ..Default::default() + }) + .expect("save should succeed"); + + let repaired = kv.get_certbot_config().expect("record should be readable"); + assert_eq!(repaired.acme_url, "https://acme-staging.example/directory"); + } + + #[test] + fn corrupt_global_records_fail_closed() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let kv = test_kv(dir.path()); + put_raw(&kv, keys::GLOBAL_ACME_ATTESTATION, b"not-messagepack"); + put_raw(&kv, keys::DNS_CRED_DEFAULT, b"not-messagepack"); + assert!(kv.get_acme_attestation().is_err()); + assert!(kv.get_default_dns_credential_id().is_err()); + assert!(kv.get_default_dns_credential().is_err()); + } + + #[test] + fn future_dated_observations_are_ignored() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let kv = test_kv(dir.path()); + let now = now_secs(); + + kv.ephemeral + .write() + .put_encoded(keys::handshake("cvm", 2), &(now.saturating_sub(30)), false) + .unwrap(); + kv.ephemeral + .write() + .put_encoded(keys::handshake("cvm", 3), &u64::MAX, false) + .unwrap(); + // A peer with a broken clock must not keep a dead CVM alive forever. + let latest = kv + .get_instance_latest_handshake("cvm") + .expect("the plausible observation should survive"); + assert!(latest <= now, "kept a future-dated handshake: {latest}"); + assert_eq!(kv.get_instance_handshakes("cvm").len(), 1); + + kv.ephemeral + .write() + .put_encoded(keys::last_seen_node(7, 3), &u64::MAX, false) + .unwrap(); + assert_eq!(kv.get_node_latest_last_seen(7), None); + assert!(kv.get_node_last_seen_by_all(7).is_empty()); + + // Drift within the allowance stays usable: nodes are not perfectly + // synchronized and dropping every slightly-ahead record would make + // instances look stale. + kv.ephemeral + .write() + .put_encoded( + keys::handshake("cvm", 4), + &(now + MAX_CLOCK_DRIFT_SECS / 2), + false, + ) + .unwrap(); + assert_eq!(kv.get_instance_handshakes("cvm").len(), 2); + } + + #[test] + fn a_storage_failure_fails_the_boot_instead_of_quarantining_intact_state() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + // Stand in for the storage being unusable — a full disk, an exhausted + // fd table, a volume that has not finished mounting. None of these say + // anything about the contents, and the condition survives a restart, so + // quarantining here would discard intact state once per boot attempt. + let data_dir = dir.path().join("kv"); + std::fs::write(&data_dir, b"not a directory").expect("failed to create blocker"); + + let Err(err) = KvStore::new(1, vec![], &data_dir, None) else { + panic!("startup must fail when the storage is unusable"); + }; + assert!( + format!("{err:#}").contains("refusing to start"), + "wrong failure: {err:#}" + ); + let quarantined = std::fs::read_dir(dir.path()) + .expect("failed to read temp dir") + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.file_name().to_string_lossy().contains(".corrupt.")) + .count(); + assert_eq!(quarantined, 0, "quarantined a directory it could not read"); + } + + #[test] + fn an_unreadable_data_dir_is_quarantined_instead_of_blocking_startup() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let data_dir = dir.path().join("kv"); + { + let kv = test_kv(&data_dir); + kv.sync_instance( + "cvm", + &InstanceRecord { + app_id: "app".to_string(), + ip: "10.0.0.20".parse().unwrap(), + public_key: "key".to_string(), + reg_time: 1, + port_policy: None, + port_policy_hash: String::new(), + health_check: None, + }, + ) + .expect("sync should succeed"); + kv.persist_if_dirty().expect("persist should succeed"); + } + // A torn WAL tail is the normal artifact of a crash and every record is + // replicated, so it must not keep the gateway from booting. + std::fs::write(data_dir.join("node_1.wal"), b"garbage").expect("failed to corrupt wal"); + + let kv = + KvStore::new(1, vec![], &data_dir, None).expect("startup must survive a corrupt wal"); + let loaded = kv.load_all_instances(); + assert!(loaded.decoded.is_empty()); + assert!(loaded.undecodable.is_empty()); + let quarantined: Vec<_> = std::fs::read_dir(dir.path()) + .expect("failed to read temp dir") + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.file_name().to_string_lossy().contains(".corrupt.")) + .collect(); + assert_eq!( + quarantined.len(), + 1, + "the unreadable data dir must be kept for inspection" + ); + } + + #[test] + fn a_cert_config_that_disagrees_with_its_key_is_skipped() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let kv = test_kv(dir.path()); + kv.save_zt_domain_config(&ZtDomainConfig { + domain: "good.example".to_string(), + dns_cred_id: None, + port: 443, + node: None, + priority: 0, + challenge: Default::default(), + }) + .expect("save should succeed"); + // Same record filed under another domain's key: honouring the value + // would request a certificate for a domain nobody configured. + kv.persistent + .write() + .put_encoded( + keys::zt_domain_config("victim.example"), + &ZtDomainConfig { + domain: "attacker.example".to_string(), + dns_cred_id: None, + port: 443, + node: None, + priority: 100, + challenge: Default::default(), + }, + true, + ) + .expect("raw put should succeed"); + + let domains: Vec<String> = kv + .list_zt_domain_configs() + .into_iter() + .map(|c| c.domain) + .collect(); + assert_eq!(domains, vec!["good.example".to_string()]); + } +} + +#[cfg(test)] +mod peer_url_tests { + use super::validate_peer_url; + + #[test] + fn accepts_http_sync_urls() { + assert!(validate_peer_url("https://gateway.example:8011/sync").is_ok()); + assert!(validate_peer_url("http://127.0.0.1:8011").is_ok()); + } + + #[test] + fn rejects_malformed_or_unsafe_sync_urls() { + for url in [ + "not-a-sync-url", + "ftp://gateway.example/sync", + "https://user:secret@gateway.example/sync", + ] { + assert!(validate_peer_url(url).is_err(), "accepted {url}"); + } + } +} + +/// The key namespace is the on-disk contract between releases. +/// +/// Every builder and parser here survived mutation: `handshake_prefix` could return +/// `""`, `parse_inst_key` could return `Some("xyzzy")`, and nothing noticed. That is not +/// a cosmetic gap — these strings are what a gateway uses to find its own state after an +/// upgrade. Changing one silently orphans every existing record: the data is still +/// replicated, still in the digest, and no longer reachable by any reader. +#[cfg(test)] +mod key_schema_tests { + use super::keys; + + /// A prefix must actually be a prefix of the keys it is used to iterate, or a range + /// scan silently returns nothing and the caller reads an empty collection as "none". + #[test] + fn every_iteration_prefix_matches_the_keys_it_must_find() { + assert!(keys::handshake("inst-a", 7).starts_with(&keys::handshake_prefix("inst-a"))); + assert!(keys::last_seen_node(3, 7).starts_with(&keys::last_seen_node_prefix(3))); + assert!(keys::cert_attestation_latest("a.example") + .starts_with(&keys::cert_attestation_prefix("a.example"))); + assert!(keys::cert_attestation_history("a.example", 1234) + .starts_with(&keys::cert_attestation_prefix("a.example"))); + } + + /// A prefix must not be so short that it also matches a neighbour's keys, which + /// would make an iteration return another instance's or node's records. + #[test] + fn an_iteration_prefix_does_not_capture_a_neighbour() { + assert!(!keys::handshake("inst-b", 7).starts_with(&keys::handshake_prefix("inst-a"))); + assert!(!keys::last_seen_node(4, 7).starts_with(&keys::last_seen_node_prefix(3))); + assert!(!keys::cert_attestation_latest("b.example") + .starts_with(&keys::cert_attestation_prefix("a.example"))); + // `inst-a` must not swallow `inst-ab`. + assert!(!keys::handshake("inst-ab", 7).starts_with(&keys::handshake_prefix("inst-a"))); + } + + /// Builders and parsers must agree, or a record written by one release is invisible + /// to the next. + #[test] + fn every_key_parses_back_to_what_built_it() { + assert_eq!(keys::parse_inst_key(&keys::inst("inst-a")), Some("inst-a")); + assert_eq!(keys::parse_node_info_key(&keys::node_info(42)), Some(42)); + assert_eq!( + keys::parse_cert_domain(&keys::cert_attestation_latest("a.example")), + Some("a.example") + ); + assert_eq!( + keys::parse_cert_domain(&keys::cert_lock("a.example")), + Some("a.example") + ); + } + + /// A parser must reject a key from another namespace rather than returning a value + /// derived from it, which would cross-wire two record types. + #[test] + fn a_parser_refuses_a_key_from_another_namespace() { + assert_eq!(keys::parse_inst_key(&keys::node_info(1)), None); + assert_eq!(keys::parse_cert_domain(&keys::inst("inst-a")), None); + // The overrides and the instance record are keyed by the same id, and + // the instance record is iterated by prefix. Either namespace claiming + // the other would make an operator's decision arrive as an instance + // record, or the reverse. + assert_eq!(keys::parse_inst_key(&keys::admin_ready("inst-a")), None); + assert!(!keys::admin_ready("inst-a").starts_with(keys::INST_PREFIX)); + assert!(!keys::inst("inst-a").starts_with(keys::ADMIN_PREFIX)); + // Two overrides of one instance, and the same override of two + // instances, must all be distinct keys. + assert_ne!( + keys::admin_ready("inst-a"), + keys::admin_port_policy("inst-a") + ); + assert_ne!(keys::admin_ready("inst-a"), keys::admin_ready("inst-ab")); + assert_eq!(keys::parse_node_info_key(&keys::node_status(1)), None); + assert_eq!(keys::parse_node_info_key(&keys::inst("inst-a")), None); + // `node/info/` and `node/status/` share a stem; neither may claim the other. + assert_eq!(keys::parse_node_info_key("node/info/not-a-number"), None); + } +} + +/// Tombstone GC: what may be collected (the ack watermark), when collection +/// triggers (the replicated write count), and how the pace is shared (the KV +/// override). Split from `corruption_tests` because these are a feature's +/// tests, not corruption drills -- they only borrow its raw-write helpers. +#[cfg(test)] +mod tombstone_gc_tests { + use super::corruption_tests::{put_raw, test_kv}; + use super::*; + + fn seed_instance(kv: &KvStore, id: &str) { + kv.sync_instance( + id, + &InstanceRecord { + app_id: "app".to_string(), + ip: "10.0.0.20".parse().unwrap(), + public_key: format!("key-{id}"), + reg_time: 1, + port_policy: None, + port_policy_hash: String::new(), + health_check: None, + }, + ) + .expect("seed"); + } + + /// Whether the key is still held at all, tombstone included. + fn key_is_held(kv: &KvStore, key: &str) -> bool { + kv.persistent.read().get_including_tombstones(key).is_some() + } + + /// A delete leaves a tombstone, and nothing collected them: they are + /// replicated state, on disk, kept for the life of the deployment. A + /// single-node gateway has no peer to resurrect from, which is exactly the + /// case wavekv collects unconditionally. + #[test] + fn a_single_node_collects_the_tombstones_it_has_finished_with() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = test_kv(dir.path()); + seed_instance(&kv, "cvm"); + kv.sync_delete_instance("cvm").expect("delete"); + assert!( + key_is_held(&kv, &keys::inst("cvm")), + "a delete leaves a tombstone behind" + ); + assert!( + kv.load_all_instances().decoded.is_empty(), + "which is not the same as the record still being live" + ); + + let collected = kv.collect_tombstone_garbage().expect("collect"); + // The delete tombstones the admin override keys alongside `inst/` + // (recycled ids must not inherit an operator's gate), so the count is + // "at least the instance record", not exactly one. + assert!(collected.persistent >= 1); + assert!(!key_is_held(&kv, &keys::inst("cvm"))); + } + + /// The watermark, not a clock, is what makes this safe. A peer that has not + /// acknowledged the delete may still be holding the live record, so + /// dropping the tombstone would let it push that record back -- a + /// deregistered CVM reappearing in every node's WireGuard config. v1 took a + /// TTL here and 2.0 removed the API for this reason. + #[test] + fn a_tombstone_a_peer_has_not_acknowledged_is_kept() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = test_kv(dir.path()); + seed_instance(&kv, "cvm"); + kv.sync_delete_instance("cvm").expect("delete"); + kv.add_peer(2).expect("add peer"); + + let collected = kv.collect_tombstone_garbage().expect("collect"); + assert_eq!(collected.persistent, 0); + assert!( + key_is_held(&kv, &keys::inst("cvm")), + "a peer that has not seen the delete can still resurrect the record" + ); + } + + /// Exchange sync envelopes until a full round moves no entries in either + /// direction, then stop. + /// + /// The real path, not a hand-set ack: what makes a tombstone collectable is + /// the peer reporting that it covers the delete, and only a round trip + /// produces that report. The quiet round's envelopes are still applied -- + /// an empty delta still carries the sender's ack map, which is exactly the + /// report the collector is waiting on. + fn sync_until_quiet(left: &KvStore, right: &KvStore) { + for _ in 0..8 { + let mut moved = 0; + for (from, to) in [(left, right), (right, left)] { + let request = from + .persistent + .read() + .prepare_sync(to.my_node_id(), vec![0u8; 16]); + moved += request.entries.len(); + to.persistent + .write() + .apply_envelope(request) + .expect("peer applies our envelope"); + let reply = to + .persistent + .read() + .prepare_sync(from.my_node_id(), vec![0u8; 16]); + moved += reply.entries.len(); + from.persistent + .write() + .apply_envelope(reply) + .expect("we apply the peer's envelope"); + } + if moved == 0 { + return; + } + } + panic!("the stores were still exchanging entries after 8 rounds"); + } + + /// The property the ack watermark buys, and the one a TTL cannot: age plays + /// no part. A tombstone written a moment ago is collectable the instant + /// every peer covers the delete -- and until then it is kept no matter how + /// long that takes, where a TTL would drop it and let the peer push the + /// record it still holds back as a live value. + #[test] + fn a_tombstone_is_collected_on_coverage_not_on_age() { + let left_dir = tempfile::tempdir().expect("temp dir"); + let right_dir = tempfile::tempdir().expect("temp dir"); + let left = KvStore::new(1, vec![2], left_dir.path(), None).expect("left"); + let right = KvStore::new(2, vec![1], right_dir.path(), None).expect("right"); + + seed_instance(&left, "cvm"); + sync_until_quiet(&left, &right); + assert!( + right.load_all_instances().decoded.contains_key("cvm"), + "the peer must hold the live record, or there is nothing to resurrect" + ); + + left.sync_delete_instance("cvm").expect("delete"); + // Freshly written, and the peer has not been told. Age is irrelevant + // here; coverage is not. + assert_eq!( + left.collect_tombstone_garbage() + .expect("collect") + .persistent, + 0 + ); + assert!(key_is_held(&left, &keys::inst("cvm"))); + + sync_until_quiet(&left, &right); + assert!( + right.load_all_instances().decoded.is_empty(), + "the peer must have applied the delete" + ); + assert!( + left.collect_tombstone_garbage() + .expect("collect") + .persistent + >= 1, + "once every peer covers the delete, nothing can push the record back" + ); + assert!(!key_is_held(&left, &keys::inst("cvm"))); + + // And the peer cannot reintroduce it afterwards. + sync_until_quiet(&left, &right); + assert!(left.load_all_instances().decoded.is_empty()); + assert!(right.load_all_instances().decoded.is_empty()); + } + + /// The GC trigger counts replicated writes instead of reading a clock, so + /// what it counts must behave like replicated state: advance with writes, + /// and read the same on every converged node. + #[test] + fn the_replicated_write_count_advances_with_writes_and_converges_between_peers() { + let left_dir = tempfile::tempdir().expect("temp dir"); + let right_dir = tempfile::tempdir().expect("temp dir"); + let left = KvStore::new(1, vec![2], left_dir.path(), None).expect("left"); + let right = KvStore::new(2, vec![1], right_dir.path(), None).expect("right"); + + let before = left.replicated_writes(); + seed_instance(&left, "cvm"); + left.sync_delete_instance("cvm").expect("delete"); + let after = left.replicated_writes(); + assert!( + after.persistent > before.persistent, + "a write and a delete both advance the count" + ); + + assert!( + right.replicated_writes().persistent < after.persistent, + "the peer has not covered the writes yet" + ); + sync_until_quiet(&left, &right); + assert_eq!( + left.replicated_writes().persistent, + right.replicated_writes().persistent, + "converged nodes read the same count, which is what lets them share GC boundaries" + ); + } + + /// `RemovePeer` drops the removed origin's ack watermark, so the write + /// count steps back for good -- the regression the GC task's baseline + /// reset exists for: without it, a boundary earned before the removal + /// gates collection until the cluster re-earns writes it no longer + /// remembers. + #[test] + fn removing_a_peer_permanently_lowers_the_replicated_write_count() { + let left_dir = tempfile::tempdir().expect("temp dir"); + let right_dir = tempfile::tempdir().expect("temp dir"); + let left = KvStore::new(1, vec![2], left_dir.path(), None).expect("left"); + let right = KvStore::new(2, vec![1], right_dir.path(), None).expect("right"); + + // Writes authored by the peer, so the count left holds for origin 2 + // is exactly what the removal deletes. + seed_instance(&right, "cvm"); + sync_until_quiet(&left, &right); + let before = left.replicated_writes().persistent; + + assert!(left.peer_ids().contains(&2)); + left.remove_peer(2).expect("remove peer"); + assert!( + !left.peer_ids().contains(&2), + "the shrinkage the GC task watches for" + ); + assert!( + left.replicated_writes().persistent < before, + "the removed origin's watermark is gone from the count, permanently" + ); + } + + /// The resurrection the watermark cannot prevent, pinned as a known + /// limitation. + /// + /// Removing a peer vacates the watermark, collection then drops the + /// tombstone, and the removed node returning with its old data directory + /// still holds the record live. Ordinary rounds do not leak it back: the + /// returning node's request re-teaches the responder coverage of its + /// origin, so nothing it authored is re-sent. What breaks it is wavekv's + /// own divergence repair -- the digests disagree for as long as the + /// returning node holds the zombie, and after `digest_check_rounds` the + /// repair (`reset_peer_coverage`) makes it re-send everything, tombstoned + /// keys included, with nothing left to beat them under LWW. + /// + /// Until a removed node is locked out at the sync boundary, removal must + /// mean decommission: the removed node's data directory must never come + /// back. + #[test] + fn a_removed_node_returning_with_old_state_resurrects_collected_deletes() { + let left_dir = tempfile::tempdir().expect("temp dir"); + let right_dir = tempfile::tempdir().expect("temp dir"); + let left = KvStore::new(1, vec![2], left_dir.path(), None).expect("left"); + let right = KvStore::new(2, vec![1], right_dir.path(), None).expect("right"); + + // The record is authored by the node that will be removed. + seed_instance(&right, "cvm"); + sync_until_quiet(&left, &right); + + // right goes dark; left deletes the record, retires right, and -- now + // peerless -- collects the tombstone. + left.sync_delete_instance("cvm").expect("delete"); + left.remove_peer(2).expect("retire"); + assert!( + left.collect_tombstone_garbage() + .expect("collect") + .persistent + >= 1 + ); + assert!(left.load_all_instances().decoded.is_empty()); + assert!( + right.load_all_instances().decoded.contains_key("cvm"), + "the removed node still holds the record live -- the zombie" + ); + + // right comes back and initiates rounds -- the real topology: left + // pruned it, so left never initiates. Ordinary rounds do not + // resurrect: right's request carries its ack map, left re-adopts + // coverage of origin 2, and right's zombie stays filtered out. + let ordinary_round = || { + let request = right.persistent.read().prepare_sync(1, vec![0u8; 16]); + let reply = left + .persistent + .write() + .handle_envelope(request, vec![0u8; 16]) + .expect("left answers the returning node"); + right + .persistent + .write() + .apply_envelope(reply) + .expect("right applies the reply"); + }; + ordinary_round(); + ordinary_round(); + assert!( + left.load_all_instances().decoded.is_empty(), + "the plain delta path does not leak the zombie back" + ); + + // But the digests now disagree for good -- left lacks a key right + // holds, with no tombstone and no ack filter to reconcile them -- so + // after `digest_check_rounds` quiescent mismatches the divergence + // repair fires on the returning node, and its next request is a full + // dump. + right.persistent.write().reset_peer_coverage(1); + ordinary_round(); + assert!( + left.load_all_instances().decoded.contains_key("cvm"), + "the deregistered CVM is back, live, in every node's WireGuard config" + ); + } + + /// The pace override lives in the KV so that it is one number cluster-wide; + /// absent means "use the config-file default", and the record replicates + /// like any other write. + #[test] + fn a_tombstone_gc_override_is_absent_by_default_and_replicates() { + let left_dir = tempfile::tempdir().expect("temp dir"); + let right_dir = tempfile::tempdir().expect("temp dir"); + let left = KvStore::new(1, vec![2], left_dir.path(), None).expect("left"); + let right = KvStore::new(2, vec![1], right_dir.path(), None).expect("right"); + + assert_eq!(left.get_tombstone_gc_config().expect("read"), None); + + let config = GlobalTombstoneGcConfig { + writes_per_collection: 5000, + }; + left.set_tombstone_gc_config(&config).expect("store"); + sync_until_quiet(&left, &right); + assert_eq!( + right.get_tombstone_gc_config().expect("read"), + Some(config), + "every node reads the operator's pace, not its own default" + ); + } + + /// Fails closed like the certbot config: reading a corrupt override as the + /// per-node default would silently put this node on a different collection + /// boundary from its peers. + #[test] + fn a_corrupt_tombstone_gc_override_does_not_read_as_the_default() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = test_kv(dir.path()); + put_raw(&kv, keys::GLOBAL_TOMBSTONE_GC_CONFIG, b"not-messagepack"); + assert!(kv.get_tombstone_gc_config().is_err()); + } +} + +/// Node removal markers: the durable "this identity was retired" fact that +/// the sync lockout and peer pruning read. Live records, so the tombstone GC +/// can never collect the signal out from under either. +#[cfg(test)] +mod peer_removal_tests { + use super::corruption_tests::{put_raw, test_kv}; + use super::*; + + /// The property the marker exists for: the `__peer_addr` tombstone a + /// removal writes is collected like any other, but a fact that must + /// outlive every tombstone is stored as a live record, and collection + /// cannot touch it. + #[test] + fn a_removal_marker_survives_the_collector() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = test_kv(dir.path()); + kv.mark_peer_removed(2).expect("mark"); + // Simulate the rest of the removal: the address delete leaves a + // tombstone, and a peerless store collects everything collectable. + kv.register_peer_url(2, "https://gw2.example.com:9202") + .expect("register"); + kv.remove_peer(2).expect("drop peer"); + kv.sync_remove_node(2).expect("delete records"); + kv.collect_tombstone_garbage().expect("collect"); + + assert!( + kv.persistent + .read() + .get_including_tombstones(&keys::peer_addr(2)) + .is_none(), + "the address tombstone is gone -- the state a returning node meets" + ); + assert!( + kv.is_peer_removed(2), + "the marker is what remains to say the node was removed" + ); + } + + /// The pruning judge accepts either signal: the live marker (durable), + /// or the address tombstone (what a removal by an older binary leaves, + /// until the collector eats it). A peer with neither -- early bootstrap + /// -- is left alone. + #[test] + fn a_marked_peer_is_pruned_even_when_the_address_tombstone_is_long_gone() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = KvStore::new(1, vec![2], dir.path(), None).expect("kv"); + + // No address record, no marker: an early-bootstrap peer, kept. + kv.prune_removed_peers(); + assert!(kv.peer_ids().contains(&2)); + + // The marker alone -- the post-collection state -- prunes. + kv.mark_peer_removed(2).expect("mark"); + kv.prune_removed_peers(); + assert!( + !kv.peer_ids().contains(&2), + "the marker prunes without any tombstone to read" + ); + } + + /// Re-admission is one explicit call: clearing the marker, whose own + /// deletion is an ordinary tombstone -- once a node is welcome again + /// there is nothing that must be remembered forever. + #[test] + fn clearing_the_marker_re_admits_the_node() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = test_kv(dir.path()); + + kv.mark_peer_removed(2).expect("mark"); + assert!(kv.is_peer_removed(2)); + + assert!(kv.clear_peer_removed(2).expect("clear")); + assert!(!kv.is_peer_removed(2)); + assert!( + !kv.clear_peer_removed(2).expect("clear again"), + "idempotent, and the retry reports there was nothing to clear" + ); + } + + /// Fails closed: refusing sync from a node whose marker is unreadable is + /// recoverable by overwriting the record; admitting a removed node's full + /// dump can resurrect every collected delete. + #[test] + fn a_corrupt_removal_marker_reads_as_removed() { + let dir = tempfile::tempdir().expect("temp dir"); + let kv = test_kv(dir.path()); + put_raw(&kv, &keys::peer_removed(2), b"not-messagepack"); + assert!(kv.is_peer_removed(2)); + } +} diff --git a/dstack/gateway/src/kv/sync_service.rs b/dstack/gateway/src/kv/sync_service.rs index f691595a1..a999a4fb0 100644 --- a/dstack/gateway/src/kv/sync_service.rs +++ b/dstack/gateway/src/kv/sync_service.rs @@ -11,16 +11,19 @@ use std::sync::Arc; use anyhow::{Context, Result}; use dstack_gateway_rpc::GetPeersResponse; -use tracing::{info, warn}; +use tracing::{error, info, warn}; use wavekv::{ - sync::{ExchangeInterface, SyncConfig as KvSyncConfig, SyncManager, SyncMessage, SyncResponse}, + sync::{ + ExchangeInterface, PeerLinkStatus, SyncConfig as KvSyncConfig, SyncEnvelope, SyncManager, + SyncMessage, SyncResponse, + }, types::NodeId, Node, }; use crate::config::SyncConfig as GwSyncConfig; -use super::https_client::{HttpsClient, HttpsClientConfig}; +use super::https_client::{HttpStatusError, HttpsClient, HttpsClientConfig}; use super::KvStore; /// HTTP-based network transport for WaveKV sync. @@ -37,15 +40,42 @@ pub struct HttpSyncNetwork { } impl HttpSyncNetwork { + /// Whether a peer rejected a send with HTTP 403. + fn was_rejected(err: &anyhow::Error) -> bool { + err.chain() + .filter_map(|cause| cause.downcast_ref::<HttpStatusError>()) + .any(|status| status.0 == 403) + } + + /// Count a 403 for this node's own monitoring. Sync endpoints use 403 for + /// both removal lockouts and app-identity mismatches, so the metric and log + /// deliberately report a rejection without claiming which condition caused it. + fn note_if_rejected(err: &anyhow::Error, peer: NodeId) { + if !Self::was_rejected(err) { + return; + } + crate::metrics::record_sync_rejected(); + error!( + "peer {peer} rejected this node's sync envelope (HTTP 403); \ + this can indicate a removal lockout or an app-identity mismatch" + ); + } + /// `my_uuid` is passed in rather than read back out of the store. + /// + /// Our own uuid is local configuration, not replicated state, and sourcing + /// it from the store forced this node's `node/info` record to be written + /// before the service could be built — which is to say before `bootstrap` + /// had rebuilt the sequence counter. After a data-directory loss that made + /// the record spend a sequence number the peers already consider seen, so + /// the one record they check us against was the one guaranteed to be + /// dropped. pub fn new( kv_store: KvStore, store_path: &'static str, tls_config: &HttpsClientConfig, + my_uuid: Vec<u8>, ) -> Result<Self> { let client = HttpsClient::new(tls_config)?; - let my_uuid = kv_store - .get_peer_uuid(kv_store.my_node_id) - .context("failed to get my UUID")?; Ok(Self { client, kv_store, @@ -69,29 +99,62 @@ impl ExchangeInterface for HttpSyncNetwork { self.kv_store.get_peer_uuid(node_id) } - async fn sync_to(&self, _node: &Node, peer: NodeId, msg: SyncMessage) -> Result<SyncResponse> { - let url = self - .get_peer_url(peer) - .ok_or_else(|| anyhow::anyhow!("peer {} address not found in DB", peer))?; - - let sync_url = format!( - "{}/wavekv/sync/{}", - url.trim_end_matches('/'), - self.store_path - ); + async fn sync_to( + &self, + _node: &Node, + _peer: NodeId, + _msg: SyncMessage, + ) -> Result<SyncResponse> { + anyhow::bail!("wavekv v1 peer synchronization is not supported") + } - // Send request with msgpack + gzip encoding - // app_id verification happens during TLS handshake via AppIdVerifier - let sync_response: SyncResponse = self + /// Native WaveKV exchange. + /// + /// All deployed clusters use this wire protocol. WaveKV 1.0 data directories are + /// migrated in place during a stopped single-node upgrade; no mixed-version network + /// protocol is exposed by the gateway. + async fn sync_v2_to( + &self, + _node: &Node, + peer: NodeId, + env: SyncEnvelope, + ) -> Result<Option<SyncEnvelope>> { + let sync_url = self.route_for(peer, "sync")?; + + let body = self .client - .post_compressed_msg(&sync_url, &msg) + .post_bytes_response(&sync_url, env.encode()?) .await - .with_context(|| format!("failed to sync to peer {peer} at {sync_url}"))?; + .with_context(|| format!("failed to sync to peer {peer} at {sync_url}")) + .inspect_err(|err| Self::note_if_rejected(err, peer))?; - // Update peer last_seen on successful sync self.kv_store.update_peer_last_seen(peer); + Ok(Some(SyncEnvelope::decode(&body)?)) + } - Ok(sync_response) + /// Opportunistic push. Best-effort by design: the periodic round remains the + /// anti-entropy backstop and the only ack authority. + async fn push_to(&self, _node: &Node, peer: NodeId, env: SyncEnvelope) -> Result<()> { + let push_url = self.route_for(peer, "push")?; + self.client + .post_bytes_no_response(&push_url, env.encode()?) + .await + .with_context(|| format!("failed to push to peer {peer} at {push_url}")) + .inspect_err(|err| Self::note_if_rejected(err, peer))?; + Ok(()) + } +} + +impl HttpSyncNetwork { + fn route_for(&self, peer: NodeId, verb: &str) -> Result<String> { + let url = self + .get_peer_url(peer) + .ok_or_else(|| anyhow::anyhow!("peer {peer} address not found in DB"))?; + Ok(format!( + "{}/wavekv/{verb}/{}", + url.trim_end_matches('/'), + self.store_path + )) } } @@ -101,6 +164,17 @@ pub struct WaveKvSyncService { pub ephemeral_manager: Arc<SyncManager<HttpSyncNetwork>>, } +/// Wake the opportunistic push path after a latency-sensitive persistent write. +pub trait PersistentWriteNotifier: Send + Sync { + fn notify_persistent_write(&self); +} + +impl PersistentWriteNotifier for WaveKvSyncService { + fn notify_persistent_write(&self) { + self.persistent_manager.notify_local_write(); + } +} + impl WaveKvSyncService { /// Create a new WaveKV sync service /// @@ -108,19 +182,24 @@ impl WaveKvSyncService { /// * `kv_store` - The sync store containing persistent and ephemeral nodes /// * `sync_config` - Sync configuration /// * `tls_config` - TLS configuration for mTLS peer authentication + /// * `my_uuid` - This node's uuid, from local configuration pub fn new( kv_store: &KvStore, sync_config: &GwSyncConfig, tls_config: HttpsClientConfig, + my_uuid: Vec<u8>, ) -> Result<Self> { let sync_config = KvSyncConfig { interval: sync_config.interval, timeout: sync_config.timeout, + ..Default::default() }; // Both networks use the same persistent node for URL lookup, but different paths - let persistent_network = HttpSyncNetwork::new(kv_store.clone(), "persistent", &tls_config)?; - let ephemeral_network = HttpSyncNetwork::new(kv_store.clone(), "ephemeral", &tls_config)?; + let persistent_network = + HttpSyncNetwork::new(kv_store.clone(), "persistent", &tls_config, my_uuid.clone())?; + let ephemeral_network = + HttpSyncNetwork::new(kv_store.clone(), "ephemeral", &tls_config, my_uuid)?; let persistent_manager = Arc::new(SyncManager::with_config( kv_store.persistent().clone(), @@ -164,14 +243,30 @@ impl WaveKvSyncService { info!("WaveKV sync tasks started"); } - /// Handle incoming sync request for persistent store - pub fn handle_persistent_sync(&self, msg: SyncMessage) -> Result<SyncResponse> { - self.persistent_manager.handle_sync(msg) + fn manager_for(&self, store: &str) -> Option<&Arc<SyncManager<HttpSyncNetwork>>> { + match store { + "persistent" => Some(&self.persistent_manager), + "ephemeral" => Some(&self.ephemeral_manager), + _ => None, + } + } + + /// Handle an inbound sync envelope. + pub fn handle_envelope(&self, store: &str, env: SyncEnvelope) -> Option<Result<SyncEnvelope>> { + Some(self.manager_for(store)?.handle_envelope(env)) + } + + /// Handle an inbound opportunistic push (merges data only; never moves acks). + pub fn handle_push(&self, store: &str, env: SyncEnvelope) -> Option<Result<()>> { + Some(self.manager_for(store)?.handle_push(env)) } - /// Handle incoming sync request for ephemeral store - pub fn handle_ephemeral_sync(&self, msg: SyncMessage) -> Result<SyncResponse> { - self.ephemeral_manager.handle_sync(msg) + /// Per-peer digest and failure telemetry for both stores. + pub fn link_status(&self) -> Vec<(&'static str, Vec<PeerLinkStatus>)> { + vec![ + ("persistent", self.persistent_manager.link_status()), + ("ephemeral", self.ephemeral_manager.link_status()), + ] } } @@ -236,3 +331,45 @@ pub async fn fetch_peers_from_bootnode( Ok(()) } + +#[cfg(test)] +mod sync_rejection_tests { + use super::*; + + fn wrapped(status: u16) -> anyhow::Error { + anyhow::Error::new(HttpStatusError(status)).context( + "failed to sync to peer 2 at https://gw2.example.com:9202/wavekv/sync/persistent", + ) + } + + /// Only HTTP 403 is a rejection; transport errors and other HTTP failures + /// must not increment the rejection counter. + #[test] + fn only_a_403_reads_as_rejected() { + assert!(HttpSyncNetwork::was_rejected(&wrapped(403))); + for status in [400u16, 401, 404, 500, 503] { + assert!( + !HttpSyncNetwork::was_rejected(&wrapped(status)), + "status {status} must not read as a rejection" + ); + } + assert!(!HttpSyncNetwork::was_rejected(&anyhow::anyhow!( + "connection refused" + ))); + } + + /// The rejection must reach the node's own monitoring. + #[test] + fn a_rejection_is_counted_for_the_senders_own_monitoring() { + // Process-wide static: assert on the delta, never the absolute value. + let before = crate::metrics::sync_rejected_count(); + HttpSyncNetwork::note_if_rejected(&wrapped(500), 2); + assert_eq!( + crate::metrics::sync_rejected_count(), + before, + "an ordinary failure must not count as a rejection" + ); + HttpSyncNetwork::note_if_rejected(&wrapped(403), 2); + assert!(crate::metrics::sync_rejected_count() > before); + } +} diff --git a/dstack/gateway/src/main.rs b/dstack/gateway/src/main.rs index c91a52032..6d9a97cac 100644 --- a/dstack/gateway/src/main.rs +++ b/dstack/gateway/src/main.rs @@ -3,21 +3,18 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::{anyhow, Context, Result}; -use base64::{engine::general_purpose::STANDARD, Engine as _}; use clap::Parser; use config::{Config, TlsConfig}; -use dstack_guest_agent_rpc::{dstack_guest_client::DstackGuestClient, GetTlsKeyArgs}; -use dstack_kms_rpc::SignCertRequest; +use dstack_guest_agent_rpc::v0::{dstack_guest_client::DstackGuestClient, GetTlsKeyArgs}; use http_client::prpc::PrpcClient; -use ra_rpc::{client::RaClient, prpc_routes as prpc, rocket_helper::QuoteVerifier}; -use ra_tls::cert::{CertConfigV2, CertSigningRequestV2, Csr}; -use ra_tls::rcgen::KeyPair; +use ra_rpc::{prpc_routes as prpc, rocket_helper::QuoteVerifier}; +use ra_tls::attestation::AttestationVerifier; use rocket::{ fairing::AdHoc, figment::{providers::Serialized, Figment}, }; -use serde::{Deserialize, Serialize}; -use tracing::info; +use std::sync::Arc; +use tracing::{info, warn}; use admin_service::AdminRpcHandler; use main_service::{Proxy, ProxyOptions, RpcHandler}; @@ -32,34 +29,18 @@ mod debug_service; mod distributed_certbot; mod kv; mod main_service; +mod metrics; mod models; mod pp; mod proxy; +mod time; mod web_routes; -#[derive(Debug, Clone, Serialize, Deserialize)] -struct DebugKeyData { - /// Private key in PEM format - key_pem: String, - /// TDX quote in base64 format - quote_base64: String, - /// Event log in JSON string format - event_log: String, - /// VM config in JSON string format - vm_config: String, -} - #[global_allocator] static ALLOCATOR: jemallocator::Jemalloc = jemallocator::Jemalloc; fn app_version() -> String { - const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION"); - const VERSION: &str = git_version::git_version!( - args = ["--abbrev=20", "--always", "--dirty=-modified"], - prefix = "git:", - fallback = "unknown" - ); - format!("v{CARGO_PKG_VERSION} ({VERSION})") + dstack_build_info::app_version!() } #[derive(Parser)] @@ -101,13 +82,10 @@ async fn maybe_gen_certs(config: &Config, tls_config: &TlsConfig) -> Result<()> } } } - match config.debug.insecure_skip_attestation { - true => gen_debug_certs(config, tls_config, alt_names).await, - false => gen_prod_certs(tls_config, alt_names).await, - } + gen_certs(tls_config, alt_names).await } -async fn gen_prod_certs(tls_config: &TlsConfig, alt_names: Vec<String>) -> Result<()> { +async fn gen_certs(tls_config: &TlsConfig, alt_names: Vec<String>) -> Result<()> { info!("Using dstack guest agent for certificate generation"); let agent_client = dstack_agent().context("Failed to create dstack client")?; @@ -136,96 +114,9 @@ async fn gen_prod_certs(tls_config: &TlsConfig, alt_names: Vec<String>) -> Resul Ok(()) } -async fn gen_debug_certs( - config: &Config, - tls_config: &TlsConfig, - alt_names: Vec<String>, -) -> Result<()> { - let kms_url = config.kms_url.clone(); - if kms_url.is_empty() { - info!("KMS URL is empty, skipping cert generation"); - return Ok(()); - } - - // Check if debug key file is configured - if config.debug.key_file.is_empty() { - info!("Debug key file not configured, skipping cert generation"); - return Ok(()); - } - - // Load pre-generated key pair and quote data from JSON file - info!("Loading debug key data from: {}", config.debug.key_file); - let ctx = "Failed to read debug key, run `cargo run --bin gen_debug_key -- <simulator_url>` to generate it"; - let json_content = fs_err::read_to_string(&config.debug.key_file).context(ctx)?; - let debug_data: DebugKeyData = - serde_json::from_str(&json_content).context("Failed to parse debug key JSON")?; - - let key_pem = debug_data.key_pem; - let quote_bin = STANDARD - .decode(&debug_data.quote_base64) - .context("Failed to decode quote from base64")?; - let event_log_json = debug_data.event_log; - let vm_config_json = debug_data.vm_config; - - // Parse key pair - let key = KeyPair::from_pem(&key_pem).context("Failed to parse debug key")?; - let pubkey = key.public_key_der(); - - // Build CSR with attestation from debug quote - let attestation = - ra_tls::attestation::Attestation::from_tdx_quote(quote_bin, event_log_json.as_bytes()) - .context("Failed to create attestation from debug quote")? - .into_versioned(); - - let csr = CertSigningRequestV2 { - confirm: "please sign cert:".to_string(), - pubkey, - config: CertConfigV2 { - org_name: None, - subject: "dstack-gateway".to_string(), - subject_alt_names: alt_names, - usage_server_auth: true, - usage_client_auth: true, - ext_quote: true, - ext_app_info: true, - not_before: None, - not_after: None, - }, - attestation, - }; - let signature = csr.signed_by(&key).context("Failed to sign CSR")?; - - // Send CSR to KMS for signing - let kms_url = format!("{kms_url}/prpc"); - info!("Sending CSR to KMS for signing: {kms_url}"); - let kms_client = RaClient::new(kms_url, true).context("Failed to create kms client")?; - let kms_client = dstack_kms_rpc::kms_client::KmsClient::new(kms_client); - let sign_response = kms_client - .sign_cert(SignCertRequest { - api_version: 2, - csr: csr.to_vec(), - signature, - vm_config: vm_config_json.to_string(), - }) - .await - .context("Failed to sign certificate via KMS")?; - - let ca_cert = sign_response - .certificate_chain - .last() - .context("Empty certificate chain")? - .to_string(); - let certs = sign_response.certificate_chain.join("\n"); - - write_cert(&tls_config.mutual.ca_certs, &ca_cert)?; - write_cert(&tls_config.certs, &certs)?; - write_cert(&tls_config.key, &key.serialize_pem())?; - Ok(()) -} - fn write_cert(path: &str, cert: &str) -> Result<()> { info!("Writing cert to file: {path}"); - safe_write::safe_write(path, cert)?; + safe_write::safe_write_with_mode(path, cert, 0o600)?; Ok(()) } @@ -242,12 +133,23 @@ async fn main() -> Result<()> { let args = Args::parse(); let figment = config::load_config_figment(args.config.as_deref()); - let config = figment.focus("core").extract::<Config>()?; - + let mut config = figment.focus("core").extract::<Config>()?; // Validate node_id if config.sync.enabled && config.sync.node_id == 0 { anyhow::bail!("node_id must be greater than 0"); } + if config.debug.insecure_localhost_backend { + warn!( + "core.debug.insecure_localhost_backend = true; the app address \"localhost\" now \ + resolves to 127.0.0.1 on this host. App addresses also come from the \ + _dstack-app-address TXT record of arbitrary custom domains, so any DNS zone owner \ + can reach this host's loopback on a port of their choosing, bypassing port_policy. \ + Never use this outside local development" + ); + } + // Before anything reads `proxy.ktls`: the acceptor built later decides + // whether to extract session secrets from it. + proxy::disable_ktls_if_unsupported(&mut config.proxy); config::setup_wireguard(&config.wg)?; @@ -264,18 +166,20 @@ async fn main() -> Result<()> { set_max_ulimit()?; } - let my_app_id = if config.debug.insecure_skip_attestation { - None - } else { - let dstack_client = dstack_agent().context("Failed to create dstack client")?; - let info = dstack_client - .info() - .await - .context("Failed to get app info")?; - Some(info.app_id) - }; + // Required, not best-effort: `my_app_id` is what every peer check compares against, + // so a gateway that cannot learn its own identity must not start rather than start + // without one. A host with no guest agent fails here. + let dstack_client = dstack_agent().context("Failed to create dstack client")?; + let my_app_id = dstack_client + .info() + .await + .context("Failed to get app info")? + .app_id; let proxy_config = config.proxy.clone(); - let pccs_url = config.pccs_url.clone(); + let attestation_verifier = Arc::new( + AttestationVerifier::load(&config.attestation) + .context("failed to load attestation verifier")?, + ); let admin_auth = if config.admin.enabled { Some(admin_auth::AdminAuthFairing::from_config(&config.admin)?) } else { @@ -321,7 +225,7 @@ async fn main() -> Result<()> { }) })) .manage(state.clone()); - let verifier = QuoteVerifier::new(pccs_url); + let verifier = QuoteVerifier::new(attestation_verifier); rocket = rocket.manage(verifier); let main_srv = rocket.launch(); let admin_state = state.clone(); @@ -335,15 +239,19 @@ async fn main() -> Result<()> { } else { tracing::info!("admin server authentication enabled"); } - rocket::custom(admin_figment) + let admin_rocket = rocket::custom(admin_figment) .attach(auth_fairing) .mount("/", admin_auth::routes()) .mount("/", web_routes::routes()) .mount("/", prpc!(Proxy, AdminRpcHandler, trim: "Admin.")) .mount("/prpc", prpc!(Proxy, AdminRpcHandler, trim: "Admin.")) - .manage(admin_state) - .launch() - .await + .manage(admin_state.clone()) + .ignite() + .await?; + admin_state + .lock() + .set_admin_shutdown(admin_rocket.shutdown()); + admin_rocket.launch().await } else { std::future::pending().await } @@ -373,3 +281,34 @@ async fn main() -> Result<()> { } Ok(()) } + +#[cfg(test)] +mod startup_tests { + use super::write_cert; + use std::fs; + + #[test] + fn gateway_startup_private_file_matrix() { + let directory = tempfile::tempdir().unwrap(); + let output = directory.path().join("gateway.key"); + write_cert(output.to_str().unwrap(), "first").unwrap(); + assert_eq!(fs::read(&output).unwrap(), b"first"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + assert_eq!( + fs::metadata(&output).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + write_cert(output.to_str().unwrap(), "second").unwrap(); + assert_eq!(fs::read(&output).unwrap(), b"second"); + assert!(fs::read_dir(directory.path()).unwrap().all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .ends_with(".tmp") + })); + } +} diff --git a/dstack/gateway/src/main_service.rs b/dstack/gateway/src/main_service.rs index c44a19e9a..503bcd203 100644 --- a/dstack/gateway/src/main_service.rs +++ b/dstack/gateway/src/main_service.rs @@ -7,10 +7,10 @@ use std::{ net::Ipv4Addr, ops::Deref, sync::{Arc, Mutex, MutexGuard}, - time::{Duration, Instant, SystemTime, UNIX_EPOCH}, + time::{Duration, Instant, SystemTime}, }; -use anyhow::{bail, Context, Result}; +use anyhow::{bail, ensure, Context, Result}; use auth_client::AuthClient; use crate::distributed_certbot::DistributedCertBot; @@ -25,8 +25,7 @@ use ra_rpc::{CallContext, RpcCall, VerifiedAttestation}; use ra_tls::attestation::AppInfo; use rand::seq::IteratorRandom; use rinja::Template as _; -use safe_write::safe_write; -use serde::{Deserialize, Serialize}; +use safe_write::safe_write_with_mode; use smallvec::{smallvec, SmallVec}; use tokio::sync::{ mpsc::{unbounded_channel, UnboundedSender}, @@ -34,16 +33,19 @@ use tokio::sync::{ }; use tokio_rustls::TlsAcceptor; use tracing::{debug, error, info, warn}; +use wavekv::types::NodeId; use crate::{ cert_store::{CertResolver, CertStoreBuilder}, config::{Config, TlsConfig}, kv::{ - fetch_peers_from_bootnode, AppIdValidator, CertData, HttpsClientConfig, InstanceData, - KvStore, NodeData, NodeStatus, PortPolicy, WaveKvSyncService, + fetch_peers_from_bootnode, import, AppIdValidator, CertData, HttpsClientConfig, + InstanceRecord, KvStore, LegacyOverrides, LoadedInstances, NodeData, NodeStatus, + PortPolicy, PortPolicyOverride, ReplicatedWrites, WaveKvSyncService, }, - models::{InstanceInfo, PortPolicyView, WgConf}, + models::{InstanceInfo, PortPolicyView, ReportedCapabilities, WgConf, WgPeer}, proxy::{create_acceptor_with_cert_resolver, AddressGroup, AddressInfo, AppAddressResolver}, + time::{decode_ts, now_secs}, }; mod auth_client; @@ -67,7 +69,7 @@ pub struct ProxyInner { pub(crate) config: Arc<Config>, /// Multi-domain certbot (from KvStore DNS credentials and domain configs) pub(crate) certbot: Arc<DistributedCertBot>, - my_app_id: Option<Vec<u8>>, + my_app_id: Vec<u8>, state: Mutex<ProxyState>, pub(crate) notify_state_updated: Notify, auth_client: AuthClient, @@ -97,13 +99,24 @@ pub struct ProxyInner { const HANDSHAKE_CACHE_TTL: Duration = Duration::from_secs(30); const HANDSHAKE_REFRESH_INTERVAL: Duration = Duration::from_secs(10); -#[derive(Debug, Serialize, Deserialize, Default)] +/// The routing tables, rebuilt from WaveKV on every reload. +/// +/// Held only in memory. It carried `Serialize`/`Deserialize` from the days +/// before WaveKV, when it was written to a state file; nothing has serialized +/// it since, and the derives kept every `#[serde]` attribute on the types it +/// reaches looking load-bearing when none of them were. +#[derive(Debug, Default)] pub(crate) struct ProxyStateMut { pub(crate) apps: BTreeMap<String, BTreeSet<String>>, pub(crate) instances: BTreeMap<String, InstanceInfo>, pub(crate) allocated_addresses: BTreeSet<Ipv4Addr>, - #[serde(skip)] pub(crate) top_n: BTreeMap<String, (AddressGroup, Instant)>, + /// When each app was last warned that its whole fleet reads unhealthy. + /// + /// Dropped wherever an app is, so the bound is the number of apps that + /// exist rather than the number this process has ever seen. See + /// [`warn_all_unhealthy`]. + pub(crate) all_unhealthy_warned: BTreeMap<String, Instant>, } pub(crate) struct ProxyState { @@ -112,17 +125,156 @@ pub(crate) struct ProxyState { /// Reference to KvStore for syncing changes kv_store: Arc<KvStore>, handshake_cache: Arc<LatestHandshakesCache>, + admin_shutdown: Option<rocket::Shutdown>, + /// Reason last logged for each KV instance record this node refuses, so a + /// record that stays bad is reported once rather than on every reload. + reported_rejections: BTreeMap<String, String>, + /// The same, for `admin/` records this node cannot decode. + reported_bad_overrides: BTreeMap<String, String>, } /// Options for creating a Proxy instance pub struct ProxyOptions { pub config: Config, - pub my_app_id: Option<Vec<u8>>, + /// This gateway's own app id, from the guest agent. + /// + /// Not optional: it is the only thing `authorize_peer` and `AppIdValidator` + /// compare a peer against, so a gateway that does not know it cannot decide + /// who belongs in the cluster. `main` fails to start rather than construct one. + pub my_app_id: Vec<u8>, /// TLS configuration (from Rocket's tls config) pub tls_config: TlsConfig, } +/// Outcome of an operator-initiated CVM removal. +/// +/// Both fields are false when the instance was never known (or the removal +/// already completed), which lets the operator distinguish a mistyped +/// instance_id from an actual removal. +pub struct CvmRemoval { + /// A live instance record (decodable or not) existed in WaveKV. + pub record_existed: bool, + /// The CVM was present in this node's local data plane. + pub removed_locally: bool, +} + +/// Outcome of an operator-initiated gateway node removal. +/// +/// Both fields are false when the node was never known (or the removal +/// already completed), which lets the operator distinguish a mistyped +/// node_id from an actual removal. +pub struct NodeRemoval { + /// Any of the node's records (info, status, or sync address) was live + /// in WaveKV. + pub record_existed: bool, + /// The node was still in this gateway's sync peer set. + pub removed_from_peer_set: bool, +} + +/// A refused instance record plus its local data-plane footprint. +pub struct RejectedInstanceReport { + pub rejected: import::RejectedInstance, + /// Whether the instance still holds state in this node's data plane. An + /// unusable record keeps whatever the data plane already had, so removing + /// an active instance also drops its routing. + pub active_locally: bool, +} + impl Proxy { + /// Remove one CVM by explicit operator request. + /// + /// The tombstone is written even when this node cannot decode the stored + /// record or no longer has the CVM in memory. This makes the operation an + /// idempotent recovery path for bad replicated instance records without + /// exposing arbitrary raw-KV deletion. Re-issuing it after a partial + /// failure also sweeps up any override or telemetry record left behind. + /// + /// An error from the delete means the tombstone itself was not written, so + /// the CVM is still registered cluster-wide. The local cleanup below is + /// then deliberately skipped: the record is still live in the store, and + /// the next reload would re-import the instance anyway. Anything the store + /// dropped short of that -- an `admin/` override left for a future holder + /// of the id to inherit, or a leaked telemetry key -- is logged and does + /// not fail the removal, because the CVM is gone either way and failing + /// would abort the routing cleanup an operator reached for this call to + /// get. + pub fn remove_cvm(&self, instance_id: &str) -> Result<CvmRemoval> { + let mut state = self.lock(); + let record_existed = state + .kv_store + .sync_delete_instance(instance_id) + .with_context(|| format!("failed to delete CVM {instance_id} from WaveKV"))?; + + let removed_locally = state.forget_instance(instance_id).is_some(); + // Reconfigure unconditionally: the tombstone write and the in-memory + // removal are not repeated on a retry, so gating this on them would + // leave a failed reconfigure with no retry path and the removed CVM's + // WireGuard peer stuck on the interface. + state.reconfigure()?; + Ok(CvmRemoval { + record_existed, + removed_locally, + }) + } + + /// Instance records this node currently refuses to import. + /// + /// Recomputed from the store on every call rather than read from the + /// cached rejection log, so the answer is current even right after a + /// restart and does not depend on when the last reload ran. + pub fn rejected_instances(&self) -> Vec<RejectedInstanceReport> { + let rejected = + import::accept_instances(&self.config.wg, self.kv_store.load_all_instances()).rejected; + let state = self.lock(); + rejected + .into_iter() + .map(|rejected| RejectedInstanceReport { + active_locally: state.state.instances.contains_key(&rejected.instance_id), + rejected, + }) + .collect() + } + + /// Remove a decommissioned gateway node by explicit operator request. + /// + /// Tombstones the node's replicated records and drops it from this + /// gateway's sync peer set immediately; other gateways prune their own + /// sets when the `__peer_addr` tombstone reaches them. A node removed by + /// mistake rejoins when it restarts (startup re-registers its records), + /// or via `SetNodeUrl` from any live gateway. + pub fn remove_node(&self, node_id: NodeId) -> Result<NodeRemoval> { + ensure!( + node_id != self.config.sync.node_id, + "a node cannot remove itself" + ); + // Capture membership before the first write. Every replicated write + // this function makes wakes a watcher whose prune races it -- the + // removal marker wakes the marker watcher just as surely as the + // __peer_addr tombstone wakes the address watcher -- so the answer + // must come from a read taken while no such signal exists yet, or the + // reported membership depends on scheduling. A guard test drives both + // watchers at full speed against this. + let removed_from_peer_set = self.kv_store.peer_ids().contains(&node_id); + // The durable half first: the live marker is what keeps the removed + // node out after the __peer_addr tombstone below has been collected. + self.kv_store + .mark_peer_removed(node_id) + .with_context(|| format!("failed to write the removal marker for node {node_id}"))?; + // Drop the peer here rather than leaving it to the watchers, so the + // removal is complete when this call returns. Idempotent when a + // watcher's prune wins the race, which is why the return value above + // does not come from this call. + self.kv_store.remove_peer(node_id)?; + let record_existed = self + .kv_store + .sync_remove_node(node_id) + .with_context(|| format!("failed to delete node {node_id} from WaveKV"))?; + Ok(NodeRemoval { + record_existed, + removed_from_peer_set, + }) + } + pub async fn new(options: ProxyOptions) -> Result<Self> { let (port_policy_tx, port_policy_rx) = unbounded_channel(); let inner = ProxyInner::new(options, port_policy_tx).await?; @@ -139,6 +291,19 @@ impl ProxyInner { self.state.lock().or_panic("Failed to lock AppState") } + /// WireGuard handshake ages, without taking the routing lock. + /// + /// The cache is its own synchronization, so a caller that only needs + /// handshakes has no reason to hold `ProxyState` while this builds its map + /// -- which clones every peer's public key, and on a cold cache shells out + /// to `wg show` synchronously. + pub(crate) fn latest_handshakes( + &self, + stale_timeout: Option<Duration>, + ) -> Result<BTreeMap<String, (u64, Duration)>> { + self.handshake_cache.latest(stale_timeout) + } + pub async fn new( options: ProxyOptions, port_policy_tx: UnboundedSender<String>, @@ -152,8 +317,13 @@ impl ProxyInner { // Initialize WaveKV store without peers (peers will be added dynamically from bootnode) let kv_store = Arc::new( - KvStore::new(config.sync.node_id, vec![], &config.sync.data_dir) - .context("failed to initialize WaveKV store")?, + KvStore::new( + config.sync.node_id, + vec![], + &config.sync.data_dir, + (!config.sync.wal_sync_window.is_zero()).then_some(config.sync.wal_sync_window), + ) + .context("failed to initialize WaveKV store")?, ); info!( "WaveKV store initialized: node_id={}, sync_enabled={}", @@ -161,16 +331,42 @@ impl ProxyInner { ); // Load state from WaveKV + if kv_store.is_peer_removed(config.sync.node_id) { + // Best-effort: a removed node usually never receives its own + // marker -- the refusals the marker drives are what keep it from + // replicating here -- so this only fires when the marker slipped + // in before the lockout took effect. The reliable signal is the + // sender side reading 403 off its own sync attempts. Only a + // warning, because this copy may also be stale: re-admission + // clears the marker on the peers, which is where the lockout is + // enforced, and a node whose marker really is current gets every + // envelope refused and can do no harm either way. + warn!( + "this node's own data directory says an operator removed it from the cluster; \ + peers will refuse to sync until it is re-admitted via SetNodeUrl" + ); + } let instances = kv_store.load_all_instances(); let nodes = kv_store.load_all_nodes(); info!( - "Loaded state from WaveKV: {} instances, {} nodes", - instances.len(), + "Loaded state from WaveKV: {} instances ({} unreadable), {} nodes", + instances.decoded.len(), + instances.undecodable.len(), nodes.len() ); - let state = build_state_from_kv_store(instances); - - // Sync this node to KvStore + // Read-only: nothing may be written before the bootstrap below, so the + // move of any legacy overrides waits for the first reload. + let legacy_overrides = kv_store.legacy_instance_overrides(); + let state = build_state_from_kv_store(&config, &kv_store, instances, &legacy_overrides); + + // This node's own records are written *after* the bootstrap below, not + // here. A local write allocates a sequence number, and after a + // data-directory loss this node has no record of which numbers it + // already spent — only its peers do. `bootstrap` rebuilds the counter + // from their coverage, so anything written before it reuses numbers the + // peers already treat as seen and is silently dropped cluster-wide. + // That would strand exactly the records recovery depends on: the fresh + // uuid peers check us against, and our sync address. let node_data = NodeData { uuid: config.uuid(), url: config.sync.my_url.clone(), @@ -178,24 +374,10 @@ impl ProxyInner { wg_endpoint: config.wg.endpoint.clone(), wg_ip: config.wg.ip.to_string(), }; - if let Err(err) = kv_store.sync_node(config.sync.node_id, &node_data) { - error!("Failed to sync this node to KvStore: {err:?}"); - } - // Set this node's status to Online - if let Err(err) = kv_store.set_node_status(config.sync.node_id, NodeStatus::Up) { - error!("Failed to set node status: {err:?}"); - } - // Register this node's sync URL in DB (for peer discovery) - if let Err(err) = kv_store.register_peer_url(config.sync.node_id, &config.sync.my_url) { - error!("Failed to register peer URL: {err:?}"); - } - // Build HttpsClientConfig for mTLS communication let https_config = { let tls = &tls_config; - let cert_validator = my_app_id - .clone() - .map(|app_id| Arc::new(AppIdValidator::new(app_id)) as _); + let cert_validator = Arc::new(AppIdValidator::new(my_app_id.clone())); HttpsClientConfig { cert_path: tls.certs.clone(), key_path: tls.key.clone(), @@ -220,7 +402,12 @@ impl ProxyInner { // Create WaveKV sync service (only if sync is enabled) let wavekv_sync = if config.sync.enabled { - match WaveKvSyncService::new(&kv_store, &config.sync, https_config.clone()) { + match WaveKvSyncService::new( + &kv_store, + &config.sync, + https_config.clone(), + node_data.uuid.clone(), + ) { Ok(sync_service) => Some(Arc::new(sync_service)), Err(err) => { error!("Failed to create WaveKV sync service: {err:?}"); @@ -244,6 +431,9 @@ impl ProxyInner { state, kv_store: kv_store.clone(), handshake_cache: handshake_cache.clone(), + admin_shutdown: None, + reported_rejections: BTreeMap::new(), + reported_bad_overrides: BTreeMap::new(), }); let auth_client = AuthClient::new(config.auth.clone()); // Bootstrap WaveKV first if sync is enabled, so certbot can load certs from peers @@ -254,6 +444,20 @@ impl ProxyInner { } } + // Publish this node's own records now that the sequence counter reflects + // whatever the peers already know we have spent (see the note above). + if let Err(err) = kv_store.sync_node(config.sync.node_id, &node_data) { + error!("Failed to sync this node to KvStore: {err:?}"); + } + // Set this node's status to Online + if let Err(err) = kv_store.set_node_status(config.sync.node_id, NodeStatus::Up) { + error!("Failed to set node status: {err:?}"); + } + // Register this node's sync URL in DB (for peer discovery) + if let Err(err) = kv_store.register_peer_url(config.sync.node_id, &config.sync.my_url) { + error!("Failed to register peer URL: {err:?}"); + } + // Create CertResolver and load certificates from KvStore let cert_resolver = Arc::new(CertResolver::new()); let all_cert_data = kv_store.load_all_cert_data(); @@ -280,10 +484,7 @@ impl ProxyInner { })?; let key_pem = std::fs::read_to_string(cert_key) .with_context(|| format!("failed to read proxy cert_key {}", cert_key.display()))?; - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); + let now = now_secs(); let cert_data = CertData { cert_pem, key_pem, @@ -301,6 +502,9 @@ impl ProxyInner { let certbot = Arc::new(DistributedCertBot::new( kv_store.clone(), cert_resolver.clone(), + wavekv_sync + .clone() + .map(|service| service as Arc<dyn crate::kv::PersistentWriteNotifier>), )); // Initialize any configured domains if let Err(err) = certbot.init_all().await { @@ -323,6 +527,7 @@ impl ProxyInner { AppAddressResolver::new( config.proxy.app_address_ns_prefix.clone(), config.proxy.app_address_ns_compat, + config.proxy.app_address_dns_servers.clone(), ) .context("failed to create app address resolver")?, ); @@ -350,8 +555,8 @@ impl ProxyInner { &self.kv_store } - pub(crate) fn my_app_id(&self) -> Option<&[u8]> { - self.my_app_id.as_deref() + pub(crate) fn my_app_id(&self) -> &[u8] { + &self.my_app_id } } @@ -373,6 +578,7 @@ impl Proxy { start_cert_store_watch_task(self.clone()); start_zt_domain_watch_task(self.clone()); start_bootnode_discovery_task(self.clone()); + crate::proxy::health_check::spawn_poller(self.clone()); Ok(()) } @@ -416,6 +622,12 @@ impl Proxy { } } + pub(crate) async fn rotate_acme_credentials( + &self, + ) -> Result<crate::distributed_certbot::RotationOutcome> { + self.certbot.rotate_acme_credentials().await + } + /// Get ACME info for all managed domains (or a specific domain) pub(crate) fn acme_info(&self, domain: Option<&str>) -> Result<AcmeInfoResponse> { let kv_store = self.kv_store.clone(); @@ -432,10 +644,23 @@ impl Proxy { .collect(), }; - // Get account_uri, account_quote and account_attestation from global ACME attestation - let (account_uri, account_quote, account_attestation) = kv_store + // The account URI comes from the published credentials; the attestation + // record is written best-effort and may lag behind a rotation, so it + // only supplies the quote when it matches the current account. + let attestation = kv_store .get_acme_attestation() - .map(|att| (att.account_uri, att.quote, att.attestation)) + .context("failed to read the ACME account attestation")?; + let account_uri = kv_store + .get_acme_credentials() + .context("call RotateAcmeCredentials to replace the stored ACME credentials")? + .and_then(|creds| { + crate::distributed_certbot::extract_account_uri(&creds.acme_credentials) + }) + .or_else(|| attestation.as_ref().map(|att| att.account_uri.clone())) + .unwrap_or_default(); + let (account_quote, account_attestation) = attestation + .filter(|att| att.account_uri == account_uri) + .map(|att| (att.quote, att.attestation)) .unwrap_or_default(); for domain in &domains { @@ -470,7 +695,7 @@ impl Proxy { instance_id: &str, client_public_key: &str, compose_hash: &str, - port_policy: Option<PortPolicy>, + reported: ReportedCapabilities, ) -> Result<RegisterCvmResponse> { let mut state = self.lock(); @@ -486,16 +711,15 @@ impl Proxy { if instance_id.is_empty() { bail!("[{instance_id}] instance id is empty"); } - if client_public_key.is_empty() { - bail!("[{instance_id}] client public key is empty"); - } + import::validate_wg_public_key(client_public_key) + .with_context(|| format!("[{instance_id}] invalid client public key"))?; let client_info = state .new_client_by_id( instance_id, app_id, client_public_key, compose_hash, - port_policy, + reported, ) .context("failed to allocate IP address for client")?; if let Err(err) = state.reconfigure() { @@ -539,28 +763,191 @@ impl Proxy { } } -fn build_state_from_kv_store(instances: BTreeMap<String, InstanceData>) -> ProxyStateMut { +/// Log the records a KV import refused, one line each. +/// +/// A refused record makes its CVM invisible to this node, so it must never be +/// a silent skip. +fn report_rejected_instances(rejected: &[import::RejectedInstance]) { + for import::RejectedInstance { + instance_id, + reason, + .. + } in rejected + { + error!("ignoring KV instance record {instance_id}: {reason:#}"); + } +} + +/// Report refused records, but only what has changed since the last reload. +/// +/// A record is refused because of what it contains, so nothing about the next +/// reload will make it acceptable — it stays refused until someone rewrites it. +/// Logging the whole set every round turns one stuck record into an unbounded +/// stream of identical `error!` lines, at whatever rate peer syncs happen to +/// wake the watch task, which buries the very first occurrence. Report a record +/// when it starts being refused or its reason changes, and again when it +/// recovers, so the log carries transitions instead of a level. +fn report_new_rejections( + reported: &mut BTreeMap<String, String>, + rejected: &[import::RejectedInstance], +) { + let mut current = BTreeMap::new(); + for import::RejectedInstance { + instance_id, + reason, + .. + } in rejected + { + let reason = format!("{reason:#}"); + if reported.get(instance_id) != Some(&reason) { + error!("ignoring KV instance record {instance_id}: {reason}"); + } + current.insert(instance_id.clone(), reason); + } + for instance_id in reported.keys() { + if !current.contains_key(instance_id) { + info!("KV instance record {instance_id} is usable again"); + } + } + *reported = current; +} + +/// The data plane's view of an instance: what the CVM reported in `data`, plus +/// what an operator decided about it under `admin/`. +/// +/// One conversion for both the startup build and the reload. They had a +/// hand-written copy each, which is how `admin_port_policy` came to be dropped +/// once already -- the compiler says nothing when a field is missing from one +/// of two literals that are supposed to agree. +/// +/// The overrides are read per key, because they are separate keys: one of them +/// existing says nothing about the other. Asking per instance is what would +/// drop the port-policy override of an instance whose gate had already been +/// moved across. A key that exists answers, including when it says "cleared"; +/// only a key that does not exist falls back to the copy an older build left +/// in the instance record, or clearing an override would be undone by it. +/// +/// Returns why the overrides could not be read, if they could not. That is not +/// "nothing is overridden": the gate is an operator saying an instance must not +/// be given work, so an unreadable answer keeps it out of app-id rotation until +/// the record is readable again. Instance-id routing is never gated, so the +/// instance stays reachable for investigation either way. +fn instance_info_from_record( + store: &KvStore, + instance_id: String, + data: InstanceRecord, + legacy: Option<&LegacyOverrides>, + known_gated: bool, +) -> (InstanceInfo, Option<String>) { + let mut unreadable = None; + let (admin_port_policy, ready) = match read_overrides(store, &instance_id, legacy) { + Ok(overrides) => overrides, + Err(err) => { + unreadable = Some(format!("{err:#}")); + (None, Some(false)) + } + }; + let info = InstanceInfo { + id: instance_id, + app_id: data.app_id, + ip: data.ip, + public_key: data.public_key, + reg_time: decode_ts(data.reg_time), + port_policy: data.port_policy, + port_policy_hash: data.port_policy_hash, + admin_port_policy, + ready, + // A record written before this field existed says nothing about the + // app's intent, so it falls back to `known_gated` -- what this node + // already believes -- rather than reading as "opted out". Reading + // `false` there would reset health to `Ungated` and drop the app's + // cached selection on a record that never claimed the app opted out. + health: data.health_check.unwrap_or(known_gated).into(), + connections: Default::default(), + }; + (info, unreadable) +} + +fn read_overrides( + store: &KvStore, + instance_id: &str, + legacy: Option<&LegacyOverrides>, +) -> Result<(Option<PortPolicy>, Option<bool>)> { + let ready = match store.instance_gate(instance_id)? { + Some(ready) => Some(ready), + None => legacy.and_then(|legacy| legacy.ready), + }; + let port_policy = match store.instance_port_policy_override(instance_id)? { + Some(PortPolicyOverride::Set(policy)) => Some(policy), + Some(PortPolicyOverride::Cleared) => None, + None => legacy.and_then(|legacy| legacy.admin_port_policy.clone()), + }; + Ok((port_policy, ready)) +} + +/// Report override records this node cannot read, once per transition. +/// +/// Unreadable means the instance is held out of app-id rotation, so it is not a +/// quiet condition -- but a record that stays bad would otherwise be reported on +/// every reload, and reloads are driven by cluster activity. +fn report_unreadable_overrides( + reported: &mut BTreeMap<String, String>, + unreadable: BTreeMap<String, String>, +) { + for (instance_id, reason) in &unreadable { + if reported.get(instance_id) != Some(reason) { + error!( + "cannot read the operator overrides for instance {instance_id}, holding it \ + out of load balancing until they are readable again: {reason}" + ); + } + } + for instance_id in reported.keys() { + if !unreadable.contains_key(instance_id) { + info!("operator overrides for instance {instance_id} are readable again"); + } + } + *reported = unreadable; +} + +fn build_state_from_kv_store( + config: &Config, + store: &KvStore, + instances: LoadedInstances, + legacy_overrides: &BTreeMap<String, LegacyOverrides>, +) -> ProxyStateMut { let mut state = ProxyStateMut::default(); + let accepted = import::accept_instances(&config.wg, instances); + report_rejected_instances(&accepted.rejected); + // Build instances - for (instance_id, data) in instances { - let info = InstanceInfo { - id: instance_id.clone(), - app_id: data.app_id.clone(), - ip: data.ip, - public_key: data.public_key, - reg_time: UNIX_EPOCH - .checked_add(Duration::from_secs(data.reg_time)) - .unwrap_or(UNIX_EPOCH), - port_policy: data.port_policy, - port_policy_hash: data.port_policy_hash, - admin_port_policy: data.admin_port_policy, - connections: Default::default(), - }; - state.allocated_addresses.insert(data.ip); + for (instance_id, data) in accepted.instances { + let ip = data.ip; + let app_id = data.app_id.clone(); + let legacy = legacy_overrides.get(&instance_id); + // Health is this node's own observation and is deliberately not + // persisted, so a restart starts every instance at `Unknown`. That does + // not blackhole anything: `Unknown` is not healthy, so an app whose + // instances are *all* unknown trips the fail-open in `retain_healthy` + // and is routed to exactly as before, until the first round of polls + // answers a few seconds later. + // + // Nothing is in memory yet to fall back on, unlike the reload path, so + // a record predating the declaration reads as not gated: routable and + // unpolled, and the CVM's next re-registration states it again. + let (info, unreadable) = + instance_info_from_record(store, instance_id.clone(), data, legacy, false); + if let Some(reason) = unreadable { + error!( + "cannot read the operator overrides for instance {instance_id}, holding it \ + out of load balancing until they are readable again: {reason}" + ); + } + state.allocated_addresses.insert(ip); state .apps - .entry(data.app_id) + .entry(app_id) .or_default() .insert(instance_id.clone()); state.instances.insert(instance_id, info); @@ -596,7 +983,17 @@ async fn start_certbot_task(proxy: Proxy) { loop { // Get current config from KV store (allows dynamic updates) - let renew_interval = proxy.kv_store.get_certbot_config().renew_interval; + let renew_interval = match proxy.kv_store.get_certbot_config() { + Ok(config) => config.renew_interval, + Err(err) => { + // Falling back to the defaults here would switch acme_url + // back to Let's Encrypt production; wait for an operator to + // repair the record instead. + error!("failed to read certbot config, skipping renewal round: {err:?}"); + tokio::time::sleep(Duration::from_secs(60)).await; + continue; + } + }; if renew_interval.is_zero() { // Check again later if disabled tokio::time::sleep(Duration::from_secs(60)).await; @@ -752,6 +1149,143 @@ async fn start_wavekv_sync_task(proxy: Proxy, wavekv_sync: Arc<WaveKvSyncService info!("WaveKV sync tasks started"); } +/// How often the tombstone GC task re-reads its trigger. An implementation +/// detail, not a policy knob: one read of two tiny records and two ack-map +/// sums. The pace of collection itself comes from `tombstone_gc_writes`. +/// +/// Short enough that two nodes crossing the same write-count boundary do so +/// within one check period plus one sync round of each other -- well inside +/// the `digest_check_rounds` window the divergence detector needs before it +/// calls the gap a repair case. That window is `sync.interval` times +/// `digest_check_rounds` -- three minutes on the defaults -- so the margin is +/// an assumption about configuration, not a constant: an operator who +/// tightens the sync interval toward this period erodes it. +const TOMBSTONE_GC_CHECK_PERIOD: Duration = Duration::from_secs(15); + +/// Whether a store has crossed a collection boundary since the last one. +/// +/// The trigger is a pure function of the replicated write count and the shared +/// pace `N`: collect when `writes / N` has grown. Every node evaluates the +/// same function over state that converges within a sync round, so the cluster +/// crosses each boundary together **without any node reading a clock** -- the +/// wall-clock-aligned design this replaces was correct only while every host's +/// clock stayed stepped-together, an assumption this has no use for. +/// +/// `last` is the count at this node's previous collection, `None` when it has +/// not collected since the task started (or since collection was re-enabled), +/// which is read as due: the backlog case, where a store that never writes +/// again would otherwise never cross a boundary and never shed the tombstones +/// it already holds. +/// +/// In a cluster the backlog case has a cost: a restarted node collects alone, +/// and if anything was collectable the digest repair puts it back a few +/// rounds later, to be shed for good at the next shared boundary. That is one +/// bounded repair per restart, paid to keep the guarantee that a store nobody +/// ever writes to again still ends up empty. +fn tombstone_collection_due(last: Option<ReplicatedWrites>, now: ReplicatedWrites, n: u64) -> bool { + let Some(last) = last else { + return true; + }; + now.persistent / n > last.persistent / n || now.ephemeral / n > last.ephemeral / n +} + +/// Drop tombstones the cluster has finished with. +/// +/// Beside the persist and WAL tasks because it belongs to the same set: work +/// the store needs on a schedule whether or not this node has peers. A +/// single-node gateway has nobody to resurrect from, so wavekv collects every +/// tombstone it holds -- and it is the deployment where nothing else ever +/// would. +/// +/// The pace is read every period rather than once at startup because the +/// operator override lives in the KV itself and can change under a running +/// node. A corrupt override skips the round instead of falling back to the +/// config-file default: the default is per-node, and collecting on it while +/// peers honour the override is exactly the phase drift the shared pace +/// exists to prevent. +/// +/// Collection runs on the blocking pool for the same reason persist and WAL +/// sync do: it takes the store's write lock and walks the whole data map, and +/// a proxy's event loop cannot afford a worker parked on that. +fn start_tombstone_gc_task(proxy: &Proxy) { + let default_pace = proxy.config.sync.tombstone_gc_writes; + let kv_store = proxy.kv_store.clone(); + tokio::spawn(async move { + let mut last: Option<ReplicatedWrites> = None; + let mut peers = kv_store.peer_ids(); + let mut override_unreadable = false; + loop { + tokio::time::sleep(TOMBSTONE_GC_CHECK_PERIOD).await; + let pace = match kv_store.get_tombstone_gc_config() { + Ok(stored) => { + if override_unreadable { + info!("WaveKV: the tombstone GC override is readable again"); + override_unreadable = false; + } + stored.map_or(default_pace, |c| c.writes_per_collection) + } + Err(err) => { + // Corruption is deterministic until an operator overwrites + // the record, so report the pause once, not every period. + if !override_unreadable { + error!( + "WaveKV: the tombstone GC override is unreadable, \ + collection is paused until it is overwritten: {err:?}" + ); + override_unreadable = true; + } + continue; + } + }; + if pace == 0 { + // Re-enabling starts from the backlog case, deliberately. + last = None; + continue; + } + // A removed peer takes its lifetime of writes out of the count for + // good (`RemovePeer` drops `acks[removed]`). A baseline above the + // shrunken count would gate collection on the cluster re-earning + // writes it no longer remembers -- on a quiet store, indefinitely. + // Removal replicates, so every node observes it within a sync + // round of the others and their resets land clustered. + let now_peers = kv_store.peer_ids(); + if !peers.is_subset(&now_peers) { + last = None; + } + peers = now_peers; + let now = kv_store.replicated_writes(); + if !tombstone_collection_due(last, now, pace) { + continue; + } + let kv = kv_store.clone(); + match tokio::task::spawn_blocking(move || kv.collect_tombstone_garbage()).await { + Ok(Ok(collected)) => { + // On failure `last` keeps its value and the next period + // retries; only success moves the boundary. + last = Some(now); + if collected.total() > 0 { + info!( + "WaveKV: collected {} tombstones ({} persistent, {} ephemeral)", + collected.total(), + collected.persistent, + collected.ephemeral + ); + } + } + Ok(Err(err)) => error!("WaveKV: tombstone collection failed: {err:?}"), + Err(err) => error!("WaveKV: the tombstone collection task did not finish: {err}"), + } + } + }); + if default_pace == 0 { + info!("WaveKV: tombstone collection disabled by default; a stored override can enable it"); + } else { + info!( + "WaveKV: tombstone collection enabled (default pace: every {default_pace} replicated writes)" + ); + } +} + fn start_wavekv_watch_task(proxy: Proxy) -> Result<()> { let kv_store = proxy.kv_store.clone(); @@ -760,14 +1294,29 @@ fn start_wavekv_watch_task(proxy: Proxy) -> Result<()> { let store_clone = kv_store.clone(); // Register watcher first, then do initial load to avoid race condition let mut rx = store_clone.watch_instances(); + // The operator overrides live under their own prefix, so a peer opening or + // closing a traffic gate does not touch `inst/` and would never wake the + // watcher above. + let mut overrides_rx = store_clone.watch_instance_overrides(); reload_instances_from_kv_store(&proxy_clone, &store_clone) .context("Failed to initial load instances from KvStore")?; tokio::spawn(async move { loop { - if rx.changed().await.is_err() { - break; - } - info!("WaveKV: detected remote instance changes, reloading..."); + let reason = tokio::select! { + changed = rx.changed() => { + if changed.is_err() { + break; + } + "instance" + } + changed = overrides_rx.changed() => { + if changed.is_err() { + break; + } + "operator override" + } + }; + info!("WaveKV: detected remote {reason} changes, reloading..."); if let Err(err) = reload_instances_from_kv_store(&proxy_clone, &store_clone) { error!("Failed to reload instances from KvStore: {err:?}"); } @@ -792,7 +1341,47 @@ fn start_wavekv_watch_task(proxy: Proxy) -> Result<()> { } }); - // Start periodic persistence task + // Watch for peer address deletions and prune the sync peer set, so a + // node removed by an operator on any gateway stops being a sync target + // here without a restart. + let mut rx = kv_store.watch_peer_addrs(); + let kv_for_peers = kv_store.clone(); + kv_for_peers.prune_removed_peers(); + tokio::spawn(async move { + loop { + if rx.changed().await.is_err() { + break; + } + kv_for_peers.prune_removed_peers(); + } + }); + + // The removal marker prunes too. It usually replicates in the same round + // as the address tombstone, but a node that was offline for the removal + // can receive the marker alone -- the tombstone may already have been + // collected everywhere else, and a live record is the one signal that + // cannot be. + let mut rx = kv_store.watch_peer_removed(); + let kv_for_removed = kv_store.clone(); + tokio::spawn(async move { + loop { + if rx.changed().await.is_err() { + break; + } + kv_for_removed.prune_removed_peers(); + } + }); + + // Start periodic persistence task. + // + // Both this and the WAL sync below run on the blocking pool. Each ends in a + // synchronous fsync — a snapshot plus its directory entry here, the log + // there — which is tens of microseconds on an NVMe host and milliseconds on + // the virtio disk a CVM gets. On a runtime worker that would park every + // other future assigned to it for the duration, which is the one thing a + // proxy's event loop cannot afford. It does not make the store lock any + // shorter: a reader still waits on the writer. It stops one slow disk from + // being a slow gateway. let persist_interval = proxy.config.sync.persist_interval; if !persist_interval.is_zero() { let kv_store_for_persist = kv_store.clone(); @@ -800,16 +1389,61 @@ fn start_wavekv_watch_task(proxy: Proxy) -> Result<()> { let mut ticker = tokio::time::interval(persist_interval); loop { ticker.tick().await; - match kv_store_for_persist.persist_if_dirty() { - Ok(true) => info!("WaveKV: periodic persist completed"), - Ok(false) => {} // No changes to persist - Err(err) => error!("WaveKV: periodic persist failed: {err:?}"), + let kv = kv_store_for_persist.clone(); + match tokio::task::spawn_blocking(move || kv.persist_if_dirty()).await { + Ok(Ok(true)) => info!("WaveKV: periodic persist completed"), + Ok(Ok(false)) => {} // No changes to persist + Ok(Err(err)) => { + crate::metrics::record_kv_persist_failure(); + error!("WaveKV: periodic persist failed: {err:?}"); + } + Err(err) => { + crate::metrics::record_kv_persist_failure(); + error!("WaveKV: the persist task did not finish: {err}"); + } } } }); info!("WaveKV: periodic persistence enabled (interval: {persist_interval:?})"); } + // Force the write-ahead log on the window the operator configured. Nothing + // else forces it on a schedule — a snapshot does, but that is minutes apart + // — so without this the window would be a hope rather than a bound. Ticking + // at the window itself is enough to make it one: the store measures from + // its last fsync, so a write is forced at the first tick that finds the + // window elapsed, never more than one window after it landed. + let wal_sync_window = proxy.config.sync.wal_sync_window; + if wal_sync_window.is_zero() { + // Not silence: no window is the strictest setting, and an operator who + // set zero expecting to turn the work off should find out here rather + // than from a latency graph. + info!("WaveKV: no WAL sync window, every write is forced before it returns"); + } else { + let kv_store_for_wal = kv_store.clone(); + tokio::spawn(async move { + let mut ticker = tokio::time::interval(wal_sync_window); + loop { + ticker.tick().await; + let kv = kv_store_for_wal.clone(); + match tokio::task::spawn_blocking(move || kv.sync_wal_if_due()).await { + Ok(Ok(_)) => {} + Ok(Err(err)) => { + crate::metrics::record_kv_wal_sync_failure(); + error!("WaveKV: forcing the write-ahead log failed: {err:?}"); + } + Err(err) => { + crate::metrics::record_kv_wal_sync_failure(); + error!("WaveKV: the write-ahead log sync did not finish: {err}"); + } + } + } + }); + info!("WaveKV: deferred WAL sync enabled (window: {wal_sync_window:?})"); + } + + start_tombstone_gc_task(&proxy); + // Start periodic connection sync task if proxy.config.sync.sync_connections_enabled { let sync_interval = proxy.config.sync.sync_connections_interval; @@ -834,84 +1468,223 @@ fn start_wavekv_watch_task(proxy: Proxy) -> Result<()> { Ok(()) } +/// Grace period protecting a freshly registered local instance from being +/// dropped by the "gone from KV" pass. +/// +/// A registration writes to ProxyState and to the KV store under the same lock, +/// so the two cannot normally disagree — but if that write failed, the instance +/// would otherwise be evicted before the CVM's next registration refresh. +const LOCAL_REGISTRATION_GRACE: Duration = Duration::from_secs(60); + fn reload_instances_from_kv_store(proxy: &Proxy, store: &KvStore) -> Result<()> { - let instances = store.load_all_instances(); + // Before the read, so an override an older node left in an instance record + // reaches its own key while that record is still the one carrying it. + let moved = store.migrate_legacy_instance_overrides(); + if !moved.is_empty() { + info!( + "WaveKV: moved operator overrides for {} instance(s) out of the instance record: {}", + moved.len(), + moved.join(", ") + ); + } + let legacy_overrides = store.legacy_instance_overrides(); + let accepted = import::accept_instances(&proxy.config.wg, store.load_all_instances()); + // An unreadable record is not a deletion. Its instance keeps whatever the + // data plane already holds, so it must be exempt from the removal pass + // below; a record that lost an IP or key conflict is not exempt, because + // the winner owns that IP or key and the loser has to stop being routable. + let unreadable: HashSet<String> = accepted + .unreadable() + .into_iter() + .map(str::to_owned) + .collect(); + let instances = accepted.instances; let mut state = proxy.lock(); + report_new_rejections(&mut state.reported_rejections, &accepted.rejected); let mut wg_changed = false; + // Instances deleted (or recycled) on another node must stop being routable + // here too, rather than lingering until this node's own recycle timeout. + let removed: Vec<String> = state + .state + .instances + .iter() + .filter(|(id, info)| { + !instances.contains_key(*id) + && !unreadable.contains(id.as_str()) + && info.reg_time.elapsed().unwrap_or_default() > LOCAL_REGISTRATION_GRACE + }) + .map(|(id, _)| id.clone()) + .collect(); + for instance_id in removed { + info!("WaveKV: instance {instance_id} was deleted remotely, dropping it"); + state.forget_instance(&instance_id); + // The node that deleted it could only withdraw its own `conn/` and + // `handshake/` keys. Ours are ours to withdraw. + if let Err(err) = store.sync_forget_local_observations(&instance_id) { + warn!("failed to withdraw this node's observations of {instance_id}: {err:?}"); + } + wg_changed = true; + } + + let mut unreadable_overrides = BTreeMap::new(); for (instance_id, data) in instances { - let new_info = InstanceInfo { - id: instance_id.clone(), - app_id: data.app_id.clone(), - ip: data.ip, - public_key: data.public_key.clone(), - reg_time: UNIX_EPOCH - .checked_add(Duration::from_secs(data.reg_time)) - .unwrap_or(UNIX_EPOCH), - port_policy: data.port_policy.clone(), - port_policy_hash: data.port_policy_hash.clone(), - admin_port_policy: data.admin_port_policy.clone(), - connections: Default::default(), - }; + let legacy = legacy_overrides.get(&instance_id); + // What this node already believes the app declared, for a record that + // predates the field. See `instance_info_from_record`. + let known_gated = state + .state + .instances + .get(&instance_id) + .is_some_and(|existing| existing.health_check()); + let (mut new_info, unreadable) = + instance_info_from_record(store, instance_id.clone(), data, legacy, known_gated); + if let Some(reason) = unreadable { + unreadable_overrides.insert(instance_id.clone(), reason); + } - let old_ip = state.state.instances.get(&instance_id).map(|e| e.ip); - if let Some(existing) = state.state.instances.get(&instance_id) { + let existing = state.state.instances.get(&instance_id).cloned(); + if let Some(existing) = &existing { // Check if wg config needs update - if existing.public_key != data.public_key || existing.ip != data.ip { + if existing.public_key != new_info.public_key || existing.ip != new_info.ip { wg_changed = true; } - // Only update if remote is newer (based on reg_time) - if data.reg_time <= encode_ts(existing.reg_time) { - continue; - } + // WaveKV has already selected the winning value. Materialize it + // unconditionally instead of applying another LWW rule here. + new_info.connections = existing.connections.clone(); + // Health is this node's own observation and WaveKV does not carry + // it, so it has to survive a reload the same way the connection + // counter does. Re-deriving it from the record would reset every + // instance to `Unknown` on every sync round -- dropping the whole + // fleet out of rotation until the next poll, over and over. The + // conditions under which it must *not* survive live with the field. + new_info.inherit_health_from(existing); } else { wg_changed = true; } // Release old IP if it changed (prevent IP leak) - if let Some(old_ip) = old_ip { - if old_ip != data.ip { - state.state.allocated_addresses.remove(&old_ip); + if let Some(existing) = &existing { + if existing.ip != new_info.ip { + state.state.allocated_addresses.remove(&existing.ip); + } + if existing.app_id != new_info.app_id { + if let Some(app_instances) = state.state.apps.get_mut(&existing.app_id) { + app_instances.remove(&instance_id); + if app_instances.is_empty() { + state.state.apps.remove(&existing.app_id); + state.state.top_n.remove(&existing.app_id); + state.state.all_unhealthy_warned.remove(&existing.app_id); + } + } } } - state.state.allocated_addresses.insert(data.ip); + // Drop any cached selection this record could invalidate. WaveKV is how + // an operator's traffic gate reaches the other nodes, and a cached + // `top_n` computed before it arrived would keep feeding the gated + // instance for up to `cache_top_n` -- on every node except the one that + // took the RPC. + let selection_is_stale = match &existing { + Some(existing) => existing.routing_inputs_differ(&new_info), + // A record this node has not seen before is a new candidate. + None => true, + }; + if selection_is_stale { + state.state.top_n.remove(&new_info.app_id); + if let Some(existing) = &existing { + state.state.top_n.remove(&existing.app_id); + } + } + state.state.allocated_addresses.insert(new_info.ip); state .state .apps - .entry(data.app_id) + .entry(new_info.app_id.clone()) .or_default() .insert(instance_id.clone()); state.state.instances.insert(instance_id, new_info); } + report_unreadable_overrides(&mut state.reported_bad_overrides, unreadable_overrides); + if wg_changed { state.reconfigure()?; } Ok(()) } +/// WireGuard peers keyed by public key, valued by (last handshake, elapsed). +pub(crate) type Handshakes = BTreeMap<String, (u64, Duration)>; + +/// One instance still in the running for a connection. +struct Candidate { + ip: Ipv4Addr, + handshake_age: Duration, + counter: Arc<std::sync::atomic::AtomicU64>, + instance_id: String, + healthy: bool, +} + +/// Drop unhealthy candidates -- unless that would drop all of them. +/// +/// Health is inference: a probe can be misconfigured, an agent can die, a whole +/// fleet can report badly at once for a reason that has nothing to do with +/// whether the app works. Blackholing an app on the strength of that is worse +/// than sending traffic to instances that might be fine. The operator gate in +/// `is_ready` deliberately does *not* get this treatment: that one is an +/// instruction, not a guess. +/// Generic over the candidate type because the two selection paths carry +/// different ones -- `select_top_n_hosts` has already resolved each instance to +/// a `Candidate`, while `random_select_a_host` is still holding instance ids. +/// They had a copy of this rule each, which is one more than the number of +/// places a change to the fail-open policy would get applied. +fn retain_healthy<T, A: smallvec::Array<Item = T>>( + state: &mut ProxyStateMut, + app_id: &str, + candidates: &mut SmallVec<A>, + is_healthy: impl Fn(&T) -> bool, +) { + if candidates.iter().any(&is_healthy) { + candidates.retain(|candidate| is_healthy(candidate)); + } else if !candidates.is_empty() { + warn_all_unhealthy(state, app_id, candidates.len()); + } +} + +/// How often one app may report that its whole fleet looks unhealthy. +const ALL_UNHEALTHY_WARN_INTERVAL: Duration = Duration::from_secs(60); + +/// Warn that an app is being routed to on fail-open, at most once a minute. +/// +/// Rate-limited here rather than relying on a caller's cache. The top-n path +/// recomputes at most every `cache_top_n`, but `random_select_a_host` has no +/// cache and runs per connection -- and a fleet that is entirely unhealthy is +/// exactly the sustained state, so an unlimited warning would be one log line +/// per inbound connection for as long as it lasts. +/// +/// The state rides in `ProxyStateMut`, which the caller is already holding, +/// rather than in a `static`. A process-global mutex here would be a new +/// serialization point on the per-connection routing path, shared across every +/// tenant -- one app's fail-open would contend with an unrelated app's +/// connections. +fn warn_all_unhealthy(state: &mut ProxyStateMut, app_id: &str, candidates: usize) { + if let Some(at) = state.all_unhealthy_warned.get(app_id) { + if at.elapsed() < ALL_UNHEALTHY_WARN_INTERVAL { + return; + } + } + state + .all_unhealthy_warned + .insert(app_id.to_string(), Instant::now()); + warn!( + "app {app_id}: no instance reports healthy; routing to all {candidates} \ + reachable instance(s) anyway rather than refusing the app" + ); +} + impl ProxyState { fn valid_ip(&self, ip: Ipv4Addr) -> bool { - // Must be within client IP range - if !self.config.wg.client_ip_range.contains(&ip) { - return false; - } - if self.config.wg.ip.broadcast() == ip { - return false; - } - if self.config.wg.ip.addr() == ip { - return false; - } - if self - .config - .wg - .reserved_net - .iter() - .any(|net| net.contains(&ip)) - { - return false; - } - true + self.config.wg.is_valid_client_ip(ip) } fn alloc_ip(&mut self) -> Option<Ipv4Addr> { for ip in self.config.wg.client_ip_range.hosts() { @@ -927,25 +1700,125 @@ impl ProxyState { None } + /// Mutate an instance record in place, dropping any cached selection the + /// change invalidates. + /// + /// The one way to write to `instances`, because "and remember to clear + /// `top_n`" is not a rule a reviewer can enforce. Six call sites got it + /// right and the seventh -- a CVM re-registering with a new WireGuard key, + /// i.e. after a reboot -- did not, so the node that took the registration + /// kept routing to a CVM whose containers were not up for as long as + /// `cache_top_n`. That is precisely the window health gating exists to + /// close, on the node most likely to be selected, and every other node in + /// the cluster got it right via the sync path. + /// + /// `InstanceInfo::routing_inputs` is the definition of "invalidates", so + /// adding a field to the selection means editing one struct rather than + /// auditing every writer. + fn edit_instance<T>( + &mut self, + id: &str, + edit: impl FnOnce(&mut InstanceInfo) -> T, + ) -> Option<T> { + let info = self.state.instances.get_mut(id)?; + let before = info.routing_inputs().to_owned(); + let out = edit(info); + if before == info.routing_inputs() { + return Some(out); + } + let before_app_id = before.app_id; + let app_id = info.app_id.clone(); + self.state.top_n.remove(&app_id); + if before_app_id != app_id { + self.state.top_n.remove(&before_app_id); + } + Some(out) + } + fn new_client_by_id( &mut self, id: &str, app_id: &str, public_key: &str, compose_hash: &str, - port_policy: Option<PortPolicy>, + reported: ReportedCapabilities, ) -> Result<InstanceInfo> { + let ReportedCapabilities { + port_policy, + health_check, + } = reported; if id.is_empty() { bail!("instance_id is empty (no_instance_id is set?)"); } if app_id.is_empty() { bail!("app_id is empty"); } - if public_key.is_empty() { - bail!("public_key is empty"); + // Checked here as well as on the KV import path: a key `wg` refuses + // makes it reject the whole config file, so it must never enter the + // instance table in the first place. + import::validate_wg_public_key(public_key).context("invalid WireGuard public key")?; + if self + .state + .instances + .values() + .any(|instance| instance.id != id && instance.public_key == public_key) + { + bail!("WireGuard public key is already registered to another instance"); } - if let Some(existing) = self.state.instances.get_mut(id) { + // The rebuild path below reconstructs the instance from scratch -- it + // does that when the recorded IP is no longer inside the configured + // range -- and everything the registration does not re-state has to + // survive that. The operator overrides in particular live nowhere else: + // lose them and the traffic gate falls open while the port-policy + // override silently reverts to whatever the instance says about itself. + let mut previous: Option<InstanceInfo> = None; + if self + .state + .instances + .get(id) + .is_some_and(|existing| existing.app_id != app_id) + { + bail!("instance_id is already registered to a different app"); + } + // Through `edit_instance`, so the app's cached selection is dropped when + // this changes something the selection was computed from. It can: a + // reboot resets health here. Doing that without invalidating leaves + // this node -- the one the CVM registered with, and therefore the one + // most likely to route to it -- serving a pre-reboot selection for up + // to `cache_top_n`, while every other node invalidates correctly off + // the changed key arriving through sync. + let edited = self.edit_instance(id, |existing| { + // A restarted CVM's health verdict describes a process that no + // longer exists, so it has to be dropped -- but "registered again" + // does not mean "restarted". `dstack-util`'s gateway-checker + // re-registers every `REFRESH_INTERVAL` (180s) and again whenever + // the WireGuard handshake goes stale, so resetting on every + // registration would drop a healthy instance to `Unknown` -- out of + // the rotation until the next poll -- every three minutes, for a + // CVM that never went anywhere. + // + // A new WireGuard key is what distinguishes the two. The guest + // caches its key store in `/run/dstack/gateway-cache.json`, which + // is tmpfs: a boot finds no cache and generates a fresh key, while + // every refresh within that boot reuses it. A changed capability + // means a different image, which also means a boot. + // + // This leans on where that cache lives. If it ever moves onto + // persistent storage -- to keep a CVM's WireGuard IP across + // reboots, say -- a reboot stops being visible here and the reset + // has to move to an explicit boot id in `RegisterCvmRequest`. let pubkey_changed = existing.public_key != public_key; + // A caller with nothing to say about the capability -- the debug + // path -- must not be able to downgrade a real CVM's declaration. + // Doing so would reset its health, then exclude it from polling for + // good, on the strength of a request that never asked about it. + let health_check = health_check.unwrap_or(existing.health_check()); + // `set_health_check` re-derives `health` when the declaration + // changes; a reboot has to reset it even when it did not. + if pubkey_changed { + existing.reset_health(); + } + existing.set_health_check(health_check); if pubkey_changed { info!("public key changed for instance {id}, new key: {public_key}"); existing.public_key = public_key.to_string(); @@ -969,39 +1842,58 @@ impl ProxyState { if port_policy.is_some() { existing.port_policy = port_policy.clone(); } - let existing = existing.clone(); + existing.clone() + }); + if let Some(existing) = edited { if self.valid_ip(existing.ip) { // Sync existing instance to KvStore (might be from legacy state) - let data = InstanceData { - app_id: existing.app_id.clone(), - ip: existing.ip, - public_key: existing.public_key.clone(), - reg_time: encode_ts(existing.reg_time), - port_policy: existing.port_policy.clone(), - port_policy_hash: existing.port_policy_hash.clone(), - admin_port_policy: existing.admin_port_policy.clone(), - }; - if let Err(err) = self.kv_store.sync_instance(&existing.id, &data) { + let record = InstanceRecord::from(&existing); + if let Err(err) = self.kv_store.sync_instance(&existing.id, &record) { error!("failed to sync existing instance to KvStore: {err:?}"); } return Ok(existing); } info!("ip {} is invalid, removing", existing.ip); self.state.allocated_addresses.remove(&existing.ip); + previous = Some(existing); } let ip = self .alloc_ip() .context("IP pool exhausted, no available addresses in client_ip_range")?; - let host_info = InstanceInfo { - id: id.to_string(), - app_id: app_id.to_string(), - ip, - public_key: public_key.to_string(), - reg_time: SystemTime::now(), - port_policy, - port_policy_hash: compose_hash.to_string(), - admin_port_policy: None, - connections: Default::default(), + let host_info = match previous { + // Rebuilding a record that already existed. Start from it and + // overwrite only what this registration actually re-states, so a + // field added later is carried by default rather than by someone + // remembering to add a line here. `health` in particular: the + // closure above has already applied this registration's + // declaration to it, and re-deriving it here would throw away the + // reboot reset it just made. + Some(existing) => InstanceInfo { + ip, + public_key: public_key.to_string(), + reg_time: SystemTime::now(), + port_policy, + port_policy_hash: compose_hash.to_string(), + connections: Default::default(), + ..existing + }, + // First registration: no operator has had the chance to set + // anything, so every field is stated here and the compiler says so. + // A caller with nothing to say about the capability -- the debug + // path -- has no earlier declaration to inherit, so it is not gated. + None => InstanceInfo { + id: id.to_string(), + app_id: app_id.to_string(), + ip, + public_key: public_key.to_string(), + reg_time: SystemTime::now(), + port_policy, + port_policy_hash: compose_hash.to_string(), + admin_port_policy: None, + ready: None, + health: health_check.unwrap_or(false).into(), + connections: Default::default(), + }, }; self.add_instance(host_info.clone()); Ok(host_info) @@ -1029,7 +1921,11 @@ impl ProxyState { return; }; info.port_policy = Some(policy); - self.persist_instance_record(instance_id); + // Pre-existing behaviour: the lazy fetch retries on its own schedule, + // so a failed write here is logged rather than surfaced. + if let Err(err) = self.persist_instance_record(instance_id) { + error!("{err:?}"); + } } /// Snapshot view of an instance's port-policy state for inspection. @@ -1059,7 +1955,11 @@ impl ProxyState { policy.ports.len(), prev.is_some(), ); - self.persist_instance_record(instance_id); + // Pre-existing behaviour: a failed write is logged, not returned. + // Left alone here so this change stays about where the record lives. + if let Err(err) = self.persist_instance_port_policy_override(instance_id) { + error!("{err:?}"); + } Ok(()) } @@ -1072,42 +1972,290 @@ impl ProxyState { let had_override = info.admin_port_policy.take().is_some(); info!("admin cleared port_policy for instance {instance_id} (was set: {had_override})"); if had_override { - self.persist_instance_record(instance_id); + // Written, not deleted. An absent key means "no operator has + // decided anything here", which lets the legacy fallback apply -- + // so deleting it would let a copy an older node left in the + // instance record put the override straight back. + if let Err(err) = self.persist_instance_port_policy_override(instance_id) { + error!("{err:?}"); + } } Ok(()) } - /// Persist the current in-memory `InstanceInfo` snapshot to WaveKV. - fn persist_instance_record(&self, instance_id: &str) { - let Some(info) = self.state.instances.get(instance_id) else { + /// Open or close the operator traffic gate for an instance. + /// + /// Closing it removes the instance from app-id load balancing but leaves it + /// running and still reachable by instance id, which is the point: an + /// operator investigating a misbehaving instance needs it out of rotation + /// and reachable at the same time. + /// + /// Existing connections are deliberately left alone. They are already + /// bridged, and tearing them down would turn "stop sending it new work" + /// into a second, louder outage. + /// + /// Errors if the instance is not registered. + pub(crate) fn set_ready(&mut self, instance_id: &str, ready: bool) -> Result<()> { + // `edit_instance` drops the pre-gate selection cache, so the change + // applies to the next connection rather than up to `cache_top_n` later. + let Some((prev, app_id)) = self.edit_instance(instance_id, |info| { + let prev = info.is_ready(); + info.ready = Some(ready); + (prev, info.app_id.clone()) + }) else { + bail!("instance {instance_id} not found"); + }; + info!("admin set ready={ready} for instance {instance_id} (prev: {prev})"); + match self.drain_warning(instance_id, &app_id, prev, ready) { + Ok(Some(message)) => warn!("{message}"), + Ok(None) => {} + // The check needs a handshake read, which can fail on its own. That + // is not a reason to fail the gate -- it is already applied -- but + // it is a reason not to imply the app is still serving. + Err(err) => debug!( + "could not tell whether app {app_id} still has an eligible instance \ + after gating {instance_id}: {err:?}" + ), + } + // Not rolled back, but do not read that as "it took effect". The gate + // is in memory and nowhere else: it is gone on restart, and any reload + // that rebuilds instances from the store reads back the very record + // this write failed to update. How long it survives depends on when + // that next happens, which is not something to predict in an error + // message -- so the message says what is known instead. Rolling back + // here would reach the same end state sooner while throwing away the + // window in which the operator's intent is at least locally in force. + self.persist_instance_gate(instance_id).with_context(|| { + format!( + "instance {instance_id} is set to ready={ready} in memory on \ + this node only: the write to the store failed, so the setting \ + is not durable and has not been shared. Retry before relying \ + on it" + ) + })?; + Ok(()) + } + + /// Record this node's latest health observation for an instance. + /// + /// Logs only when the verdict changes. Every instance is polled every few + /// seconds, so an app that stays unhealthy for an hour should cost one + /// line, not several hundred. + pub(crate) fn record_instance_health( + &mut self, + target: &crate::proxy::health_check::Target, + observation: crate::proxy::health_check::Observation, + ) { + let instance_id = target.id.as_str(); + // `edit_instance` drops the cached selection if this write changed + // anything the selection was computed from. + let applied = self.edit_instance(instance_id, |info| { + // The record still exists, but is it the same one that was polled? + // A round takes as long as its slowest instance, and a CVM can + // reboot and re-register inside that window -- new WireGuard key, + // health reset to `Unknown`, containers not up yet. Writing a + // verdict from before that would put the previous boot's answer on + // the new one and let it serve immediately, which is exactly what + // `Unknown` exists to prevent. The IP is checked for the same + // reason: it can be reallocated to another instance entirely. + if info.public_key != target.public_key || info.ip != target.ip { + return Err(()); + } + let previous = info.health(); + if previous != observation.state { + info.set_health(observation.state); + } + Ok(previous) + }); + let previous = match applied { + // Recycled or deregistered while the round was in flight. + None => return, + Some(Err(())) => { + debug!("dropping stale health observation for instance {instance_id}"); + return; + } + Some(Ok(previous)) => previous, + }; + if previous == observation.state { return; + } + let next = observation.state.as_str(); + if observation.reason.is_empty() { + info!( + "instance {instance_id} is now {next} (was {})", + previous.as_str() + ); + } else { + info!( + "instance {instance_id} is now {next} (was {}): {}", + previous.as_str(), + observation.reason + ); + } + } + + /// Whether the WireGuard tunnel to `instance` is fresh enough to route over. + fn handshake_is_fresh(&self, instance: &InstanceInfo, handshakes: &Handshakes) -> bool { + handshakes + .get(&instance.public_key) + .is_some_and(|(_, elapsed)| *elapsed < self.config.proxy.timeouts.handshake_stale) + } + + /// Whether `instance` may receive a connection addressed to its app id. + /// + /// The one definition both selection paths and the drain warning ask. + /// They had already drifted apart: the warning looked only at the gate, + /// so an instance nobody gated but whose tunnel was dead counted as cover. + fn is_eligible(&self, instance: &InstanceInfo, handshakes: &Handshakes) -> bool { + instance.is_ready() && self.handshake_is_fresh(instance, handshakes) + } + + /// The warning an operator should see after a gate change, if any. + /// + /// Returned rather than logged so both conditions are testable, because + /// both were wrong. It counted only the gate, so draining the last live + /// instance of an app whose others were dead-but-ungated said nothing -- + /// the case the warning exists for. And it keyed on the resulting count + /// alone, so a second gate-off against an already-drained app warned + /// again, naming an instance that was never the last ready one; the retry + /// the failed-write message asks for is exactly that call. + fn drain_warning( + &self, + instance_id: &str, + app_id: &str, + prev: bool, + ready: bool, + ) -> Result<Option<String>> { + // Nothing was closed, so nothing can have been closed last. + if !prev || ready { + return Ok(None); + } + if self.count_eligible_instances(app_id)? > 0 { + return Ok(None); + } + // Not an error -- an operator may well mean to drain an app entirely -- + // but silently refusing every connection to an app is the kind of thing + // someone should be told about once. + Ok(Some(format!( + "gating instance {instance_id} leaves app {app_id} with no instance eligible \ + for load balancing; connections addressed to the app id will be refused \ + until one becomes eligible again" + ))) + } + + /// Why an app-id selection came back empty. + /// + /// The proxy learns "no candidates" as an absence, and the layer that sees + /// the absence first is the port-policy filter -- which has not looked at a + /// port yet and cannot say why. Selection is where the reason exists, so + /// the reason is reconstructed here rather than guessed at downstream. + pub(crate) fn describe_empty_selection(&self, app_id: &str) -> String { + let Some(instance_ids) = self.state.apps.get(app_id) else { + return format!("app {app_id} has no registered instances"); }; - let data = InstanceData { - app_id: info.app_id.clone(), - ip: info.ip, - public_key: info.public_key.clone(), - reg_time: encode_ts(info.reg_time), - port_policy: info.port_policy.clone(), - port_policy_hash: info.port_policy_hash.clone(), - admin_port_policy: info.admin_port_policy.clone(), + let total = instance_ids.len(); + // The random-selection fallback swallows this error, so it is a real + // way to arrive here with instances that are perfectly healthy. + let handshakes = match self.latest_handshakes(None) { + Ok(handshakes) => handshakes, + Err(err) => { + return format!( + "could not read WireGuard handshakes, so none of app {app_id}'s \ + {total} instance(s) could be checked for liveness: {err:#}" + ) + } }; - if let Err(err) = self.kv_store.sync_instance(instance_id, &data) { - error!("failed to sync instance {instance_id} to KvStore: {err:?}"); + let mut gated = 0; + let mut stale = 0; + for instance in instance_ids + .iter() + .filter_map(|id| self.state.instances.get(id)) + { + if !instance.is_ready() { + gated += 1; + } else if !self.handshake_is_fresh(instance, &handshakes) { + stale += 1; + } } + format!( + "no instance of app {app_id} is currently routable: {total} registered, \ + {gated} gated out by an operator, {stale} with no recent WireGuard handshake" + ) + } + + /// How many of an app's instances can currently be given new traffic. + fn count_eligible_instances(&self, app_id: &str) -> Result<usize> { + let Some(instances) = self.state.apps.get(app_id) else { + return Ok(0); + }; + let handshakes = self + .latest_handshakes(None) + .context("failed to read WireGuard handshakes")?; + Ok(instances + .iter() + .filter_map(|id| self.state.instances.get(id)) + .filter(|info| self.is_eligible(info, &handshakes)) + .count()) + } + + /// Persist the operator's traffic gate for `instance_id` to WaveKV. + /// + /// Separate from [`Self::persist_instance_record`], and from the port-policy + /// override beside it, because each is a separate key -- which is the point. + /// This one is written only from `Admin.SetInstanceReady`, so neither a + /// re-registration nor a peer's unsynced change to the other override can + /// overwrite it. + /// + /// Returns the store error rather than only logging it, so callers whose + /// API promises the change is durable can say otherwise when it is not. + fn persist_instance_gate(&self, instance_id: &str) -> Result<()> { + let Some(info) = self.state.instances.get(instance_id) else { + return Ok(()); + }; + let Some(ready) = info.ready else { + return Ok(()); + }; + self.kv_store + .set_instance_gate(instance_id, ready) + .with_context(|| { + format!("failed to sync the gate for instance {instance_id} to KvStore") + }) + } + + /// Persist the operator's port-policy override for `instance_id` to WaveKV. + fn persist_instance_port_policy_override(&self, instance_id: &str) -> Result<()> { + let Some(info) = self.state.instances.get(instance_id) else { + return Ok(()); + }; + let policy = info.admin_port_policy.clone(); + self.kv_store + .set_instance_port_policy_override(instance_id, policy) + .with_context(|| { + format!( + "failed to sync the port-policy override for instance {instance_id} to KvStore" + ) + }) + } + + /// Persist the current in-memory `InstanceInfo` snapshot to WaveKV. + /// + /// Returns the store error rather than only logging it, so callers whose + /// API promises the change is durable can say otherwise when it is not. + fn persist_instance_record(&self, instance_id: &str) -> Result<()> { + let Some(info) = self.state.instances.get(instance_id) else { + return Ok(()); + }; + let record = InstanceRecord::from(info); + self.kv_store + .sync_instance(instance_id, &record) + .with_context(|| format!("failed to sync instance {instance_id} to KvStore")) } fn add_instance(&mut self, info: InstanceInfo) { + self.state.top_n.remove(&info.app_id); // Sync to KvStore - let data = InstanceData { - app_id: info.app_id.clone(), - ip: info.ip, - public_key: info.public_key.clone(), - reg_time: encode_ts(info.reg_time), - port_policy: info.port_policy.clone(), - port_policy_hash: info.port_policy_hash.clone(), - admin_port_policy: info.admin_port_policy.clone(), - }; - if let Err(err) = self.kv_store.sync_instance(&info.id, &data) { + let record = InstanceRecord::from(&info); + if let Err(err) = self.kv_store.sync_instance(&info.id, &record) { error!("failed to sync instance to KvStore: {err:?}"); } @@ -1120,30 +2268,96 @@ impl ProxyState { } fn generate_wg_config(&self) -> Result<String> { + // Last check before the values leave Rust: `wg syncconf` refuses the + // entire file when one peer is malformed, so a record that somehow got + // past the import boundary must cost only its own routing. + let mut peers = Vec::with_capacity(self.state.instances.len()); + for info in self.state.instances.values() { + if let Err(err) = import::validate_wg_public_key(&info.public_key) { + error!("excluding instance {} from wg config: {err:#}", info.id); + continue; + } + if info.public_key == self.config.wg.public_key { + error!( + "excluding instance {} from wg config: public key belongs to this gateway", + info.id + ); + continue; + } + if !self.config.wg.is_routable_client_ip(info.ip) { + error!( + "excluding instance {} from wg config: ip {} is outside the wg network", + info.id, info.ip + ); + continue; + } + peers.push(WgPeer { + public_key: &info.public_key, + ip: info.ip, + }); + } let model = WgConf { private_key: &self.config.wg.private_key, listen_port: self.config.wg.listen_port, - peers: (&self.state.instances).into(), + peers, }; Ok(model.render()?) } pub(crate) fn reconfigure(&mut self) -> Result<()> { + // Every way out of here that is not a clean apply leaves the data plane + // on the routing table it already had, so they all feed one counter -- + // the early returns included. A config that cannot be rendered or + // written never reaches `wg` at all, and both call sites of this + // function only log the `Err`, so a full disk would otherwise look + // exactly like having nothing to apply. + let result = self.reconfigure_inner(); + if result.is_err() { + crate::metrics::record_wg_reconfigure(false); + } + result + } + + fn reconfigure_inner(&mut self) -> Result<()> { let wg_config = self.generate_wg_config()?; - safe_write(&self.config.wg.config_path, wg_config).context("Failed to write wg config")?; + // the rendered config carries the interface's WireGuard private key. + safe_write_with_mode(&self.config.wg.config_path, wg_config, 0o600) + .context("failed to write wg config")?; // wg setconf <interface_name> <config_path> let ifname = &self.config.wg.interface; let config_path = &self.config.wg.config_path; match cmd!(wg syncconf $ifname $config_path) { - Ok(_) => info!("wg config updated"), - Err(err) => error!("failed to set wg config: {err:?}"), + Ok(_) => { + crate::metrics::record_wg_reconfigure(true); + info!("wg config updated"); + } + Err(err) => { + // `wg syncconf` rejects the whole file when one peer stanza is + // bad, and this stays `Ok` for the caller as it always has, so + // the counter is the only signal that routing updates stopped + // reaching the data plane. + crate::metrics::record_wg_reconfigure(false); + error!("failed to set wg config: {err:?}"); + } } Ok(()) } + /// Whether health observations are allowed to affect routing at all. + /// + /// With polling switched off nothing ever leaves `Unknown`, and `Unknown` + /// is not healthy -- so the filter has to be skipped outright rather than + /// merely left unfed. Otherwise disabling the feature would drop exactly + /// the instances that opted into it, and in a fleet that also has apps + /// which did not (`Ungated`, which counts as healthy) it would drop the + /// opted-in ones permanently. + fn health_gating_enabled(&self) -> bool { + self.config.proxy.health_check.enabled + } + pub(crate) fn select_top_n_hosts(&mut self, id: &str) -> Result<AddressGroup> { - if self.config.proxy.localhost_enabled && id == "localhost" { + if self.config.debug.insecure_localhost_backend && id == "localhost" { return Ok(smallvec![AddressInfo { ip: Ipv4Addr::new(127, 0, 0, 1), counter: Default::default(), @@ -1163,15 +2377,13 @@ impl ProxyState { // fallback to random selection return Ok(self.random_select_a_host(id).unwrap_or_default()); } - let (top_n, insert_time) = self - .state - .top_n - .entry(id.to_string()) - .or_insert((SmallVec::new(), Instant::now())); - if !top_n.is_empty() && insert_time.elapsed() < self.config.proxy.timeouts.cache_top_n { - return Ok(top_n.clone()); + if let Some((top_n, insert_time)) = self.state.top_n.get(id) { + if !top_n.is_empty() && insert_time.elapsed() < self.config.proxy.timeouts.cache_top_n { + return Ok(top_n.clone()); + } } + let health_gating = self.health_gating_enabled(); let handshakes = self.latest_handshakes(None); let mut instances = match handshakes { Err(err) => { @@ -1182,29 +2394,46 @@ impl ProxyState { .iter() .filter_map(|instance_id| { let instance = self.state.instances.get(instance_id)?; + // Eligibility -- including the operator gate -- is only + // applied here, on the app-id path: the instance-id lookup + // above returns before this point, so a gated instance + // stays directly reachable. + if !self.is_eligible(instance, &handshakes) { + return None; + } + // Re-read for the sort key. `is_eligible` already proved + // it is present and fresh. let (_, elapsed) = handshakes.get(&instance.public_key)?; - Some(( - instance.ip, - *elapsed, - instance.connections.clone(), - instance.id.clone(), - )) + Some(Candidate { + ip: instance.ip, + handshake_age: *elapsed, + counter: instance.connections.clone(), + instance_id: instance.id.clone(), + healthy: !health_gating || instance.is_healthy(), + }) }) .collect::<SmallVec<[_; 4]>>(), }; - instances.sort_by(|a, b| a.1.cmp(&b.1)); + retain_healthy(&mut self.state, id, &mut instances, |candidate| { + candidate.healthy + }); + instances.sort_by(|a, b| a.handshake_age.cmp(&b.handshake_age)); instances.truncate(n); - Ok(instances + let selected: AddressGroup = instances .into_iter() - .map(|(ip, _, counter, instance_id)| AddressInfo { - ip, - counter, - instance_id, + .map(|candidate| AddressInfo { + ip: candidate.ip, + counter: candidate.counter, + instance_id: candidate.instance_id, }) - .collect()) + .collect(); + self.state + .top_n + .insert(id.to_string(), (selected.clone(), Instant::now())); + Ok(selected) } - fn random_select_a_host(&self, id: &str) -> Option<AddressGroup> { + fn random_select_a_host(&mut self, id: &str) -> Option<AddressGroup> { // Direct instance lookup first if let Some(info) = self.state.instances.get(id).cloned() { return Some(smallvec![AddressInfo { @@ -1219,21 +2448,25 @@ impl ProxyState { // Get latest handshakes to check instance health let handshakes = self.latest_handshakes(None).ok()?; - // Filter healthy instances and choose randomly among them - let healthy_instances = app_instances.iter().filter(|instance_id| { - if let Some(instance) = self.state.instances.get(*instance_id) { - // Consider instance healthy if it had a recent handshake - handshakes - .get(&instance.public_key) - .map(|(_, elapsed)| *elapsed < Duration::from_secs(300)) - .unwrap_or(false) - } else { - false - } - }); + // Health is resolved here, alongside eligibility, rather than inside + // the retain: `retain_healthy` takes `&mut self.state` to rate-limit + // its warning, so its predicate cannot also be reading instances out of + // it. + let health_gating = self.health_gating_enabled(); + // Instances the operator has left open and whose tunnel is fresh, each + // paired with whether health lets it serve. + let mut eligible = app_instances + .iter() + .filter_map(|instance_id| { + let instance = self.state.instances.get(instance_id)?; + self.is_eligible(instance, &handshakes) + .then(|| (instance_id.clone(), !health_gating || instance.is_healthy())) + }) + .collect::<SmallVec<[(String, bool); 4]>>(); + retain_healthy(&mut self.state, id, &mut eligible, |(_, healthy)| *healthy); - let selected = healthy_instances.choose(&mut rand::thread_rng())?; - self.state.instances.get(selected).map(|info| { + let (selected, _) = eligible.into_iter().choose(&mut rand::thread_rng())?; + self.state.instances.get(&selected).map(|info| { smallvec![AddressInfo { ip: info.ip, counter: info.connections.clone(), @@ -1245,32 +2478,39 @@ impl ProxyState { /// Get latest handshakes /// /// Return a map of public key to (timestamp, elapsed) - pub(crate) fn latest_handshakes( - &self, - stale_timeout: Option<Duration>, - ) -> Result<BTreeMap<String, (u64, Duration)>> { + pub(crate) fn latest_handshakes(&self, stale_timeout: Option<Duration>) -> Result<Handshakes> { self.handshake_cache.latest(stale_timeout) } - fn remove_instance(&mut self, id: &str) -> Result<()> { - let info = self - .state - .instances - .remove(id) - .context("instance not found")?; - - // Sync deletion to KvStore - if let Err(err) = self.kv_store.sync_delete_instance(id) { - error!("Failed to sync instance deletion to KvStore: {err:?}"); - } - + /// Drop an instance from the local state only. + /// + /// Used when the KV store already says the instance is gone; the syncing + /// counterpart is [`Self::remove_instance`]. + fn forget_instance(&mut self, id: &str) -> Option<InstanceInfo> { + let info = self.state.instances.remove(id)?; self.state.allocated_addresses.remove(&info.ip); + self.state.top_n.remove(&info.app_id); if let Some(app_instances) = self.state.apps.get_mut(&info.app_id) { app_instances.remove(id); if app_instances.is_empty() { self.state.apps.remove(&info.app_id); + // The rate limiter is keyed by app, so it has to be dropped + // with the app rather than with the instance: keyed on "apps + // seen since this process started" instead of "apps that + // exist", it only ever grows, and the key is tenant-chosen. + self.state.all_unhealthy_warned.remove(&info.app_id); } } + Some(info) + } + + fn remove_instance(&mut self, id: &str) -> Result<()> { + self.forget_instance(id).context("instance not found")?; + + // Sync deletion to KvStore + if let Err(err) = self.kv_store.sync_delete_instance(id) { + error!("Failed to sync instance deletion to KvStore: {err:?}"); + } Ok(()) } @@ -1324,8 +2564,21 @@ impl ProxyState { Ok(()) } - pub(crate) fn exit(&mut self) -> ! { - std::process::exit(0); + pub(crate) fn set_admin_shutdown(&mut self, shutdown: rocket::Shutdown) { + self.admin_shutdown = Some(shutdown); + } + + pub(crate) fn exit(&self, force: bool) -> Result<()> { + if force { + std::process::exit(0); + } + + let shutdown = self + .admin_shutdown + .as_ref() + .context("admin server shutdown handle is not initialized")?; + shutdown.notify(); + Ok(()) } pub(crate) fn refresh_state(&mut self) -> Result<()> { @@ -1350,10 +2603,7 @@ impl ProxyState { } // Update this node's last_seen in KvStore - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); + let now = now_secs(); if let Err(err) = self .kv_store .sync_node_last_seen(self.config.sync.node_id, now) @@ -1396,16 +2646,9 @@ impl ProxyState { self.kv_store .load_all_nodes() .into_iter() - .filter(|(id, _)| { - if !exclude_down { - return true; - } - // Exclude nodes with status "down" - match node_statuses.get(id) { - Some(NodeStatus::Down) => false, - _ => true, // Include Up or nodes without explicit status - } - }) + // Shared with the metrics sampler so the gauge and the routing + // table cannot disagree about what "active" means. + .filter(|(id, _)| !exclude_down || KvStore::node_is_active(node_statuses.get(id))) .map(|(id, node)| GatewayNodeInfo { id, uuid: node.uuid, @@ -1419,16 +2662,6 @@ impl ProxyState { } } -fn decode_ts(ts: u64) -> SystemTime { - UNIX_EPOCH - .checked_add(Duration::from_secs(ts)) - .unwrap_or(UNIX_EPOCH) -} - -pub(crate) fn encode_ts(ts: SystemTime) -> u64 { - ts.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs() -} - pub struct RpcHandler { remote_app_id: Option<Vec<u8>>, remote_app_info: Option<AppInfo>, @@ -1438,13 +2671,10 @@ pub struct RpcHandler { impl RpcHandler { fn ensure_from_gateway(&self) -> Result<()> { - if self.state.config.debug.insecure_skip_attestation { - return Ok(()); - } if self.remote_app_id.is_none() { bail!("Client authentication is required"); } - if self.state.my_app_id != self.remote_app_id { + if Some(&self.state.my_app_id) != self.remote_app_id.as_ref() { bail!("Remote app id is not from dstack-gateway"); } Ok(()) @@ -1471,28 +2701,37 @@ impl GatewayRpc for RpcHandler { let app_id = hex::encode(&app_info.app_id); let instance_id = hex::encode(&app_info.instance_id); let compose_hash = hex::encode(&app_info.compose_hash); - let port_policy = request.port_policy.map(|p| { - let ports = p - .ports - .into_iter() - .filter_map(|attr| { - // Wire format is uint32 to avoid varint shenanigans, but valid TCP - // ports fit in u16. Drop out-of-range entries instead of truncating. - let port = u16::try_from(attr.port).ok()?; - Some((port, crate::kv::PortFlags { pp: attr.pp })) + let port_policy = request + .port_policy + .map(|policy| -> Result<PortPolicy> { + let ports = policy + .ports + .into_iter() + .map(|attr| { + let port = u16::try_from(attr.port) + .with_context(|| format!("port {} out of u16 range", attr.port))?; + if port == 0 { + bail!("port must be between 1 and 65535"); + } + Ok((port, crate::kv::PortFlags { pp: attr.pp })) + }) + .collect::<Result<_>>()?; + Ok(PortPolicy { + ports, + restrict_mode: policy.restrict_mode, }) - .collect(); - PortPolicy { - ports, - restrict_mode: p.restrict_mode, - } - }); + }) + .transpose()?; self.state.do_register_cvm( &app_id, &instance_id, &request.client_public_key, &compose_hash, - port_policy, + ReportedCapabilities { + port_policy, + // A real CVM always states its intent, either way. + health_check: Some(request.health_check), + }, ) } diff --git a/dstack/gateway/src/main_service/handshakes.rs b/dstack/gateway/src/main_service/handshakes.rs index 06d43ecda..a2876dd47 100644 --- a/dstack/gateway/src/main_service/handshakes.rs +++ b/dstack/gateway/src/main_service/handshakes.rs @@ -46,9 +46,44 @@ impl LatestHandshakesCache { Ok(()) } + #[cfg(test)] + pub(crate) fn set_for_test(&self, timestamps: HandshakeTimestamps) { + self.cell.set(timestamps); + } + pub(crate) fn latest(&self, stale_timeout: Option<Duration>) -> Result<HandshakesWithAge> { - let snapshot = self.cell.get()?; - add_elapsed_time(snapshot.value(), stale_timeout) + // Admin/public status paths call this synchronously. On fixture hosts the + // first successful `wg show` may not have completed yet (or the interface + // may be absent), so a hard Empty error collapses many Admin.* RPCs with + // "cached cell is empty". Prefer: + // 1) fresh TTL value + // 2) stale last-known value + // 3) one synchronous producer refresh + // 4) empty map so callers can still report registered hosts/meta + let timestamps = match self.cell.get() { + Ok(snapshot) => snapshot.into_value(), + Err(cached_cell::GetError::Expired { .. }) | Err(cached_cell::GetError::Empty) => { + match self.cell.get_allow_stale() { + Ok(snapshot) => snapshot.into_value(), + Err(_) => { + let interface = self.interface.clone(); + match fetch_latest_handshake_timestamps(&interface) { + Ok(value) => { + self.cell.set(value.clone()); + std::sync::Arc::new(value) + } + Err(err) => { + warn!( + "WireGuard latest-handshakes unavailable; returning empty map: {err}" + ); + std::sync::Arc::new(BTreeMap::new()) + } + } + } + } + } + }; + add_elapsed_time(timestamps.as_ref(), stale_timeout) } } diff --git a/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-2.snap b/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-2.snap index 78baa9249..f126f03ee 100644 --- a/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-2.snap +++ b/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-2.snap @@ -7,7 +7,7 @@ InstanceInfo { id: "test-id-1", app_id: "app-id-1", ip: 10.0.0.3, - public_key: "test-pubkey-1", + public_key: "dGVzdC1wdWJrZXktMXRlc3QtcHVia2V5LTF0ZXN0LXA=", reg_time: SystemTime { tv_sec: 0, tv_nsec: 0, @@ -15,5 +15,10 @@ InstanceInfo { port_policy: None, port_policy_hash: "", admin_port_policy: None, + ready: None, + health: Health { + gated: false, + state: Ungated, + }, connections: 0, } diff --git a/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-3.snap b/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-3.snap index df39fb280..0f9e2a19c 100644 --- a/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-3.snap +++ b/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-3.snap @@ -10,11 +10,11 @@ ListenPort = 51820 [Peer] -PublicKey = test-pubkey-0 +PublicKey = dGVzdC1wdWJrZXktMHRlc3QtcHVia2V5LTB0ZXN0LXA= AllowedIPs = 10.0.0.2/32 PersistentKeepalive = 25 [Peer] -PublicKey = test-pubkey-1 +PublicKey = dGVzdC1wdWJrZXktMXRlc3QtcHVia2V5LTF0ZXN0LXA= AllowedIPs = 10.0.0.3/32 PersistentKeepalive = 25 diff --git a/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config.snap b/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config.snap index 1f03bd3fb..221032272 100644 --- a/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config.snap +++ b/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config.snap @@ -7,7 +7,7 @@ InstanceInfo { id: "test-id-0", app_id: "app-id-0", ip: 10.0.0.2, - public_key: "test-pubkey-0", + public_key: "dGVzdC1wdWJrZXktMHRlc3QtcHVia2V5LTB0ZXN0LXA=", reg_time: SystemTime { tv_sec: 0, tv_nsec: 0, @@ -15,5 +15,10 @@ InstanceInfo { port_policy: None, port_policy_hash: "", admin_port_policy: None, + ready: None, + health: Health { + gated: false, + state: Ungated, + }, connections: 0, } diff --git a/dstack/gateway/src/main_service/tests.rs b/dstack/gateway/src/main_service/tests.rs index ec7f87831..8cf0dd52e 100644 --- a/dstack/gateway/src/main_service/tests.rs +++ b/dstack/gateway/src/main_service/tests.rs @@ -5,7 +5,11 @@ use super::*; use crate::config::{load_config_figment, Config, MutualConfig}; use crate::kv::PortFlags; +use crate::models::HealthState; use crate::proxy::port_policy::is_port_allowed; +use crate::time::encode_ts; +use base64::Engine as _; +use std::sync::atomic::Ordering; use tempfile::TempDir; struct TestState { @@ -21,13 +25,25 @@ impl std::ops::Deref for TestState { } async fn create_test_state() -> TestState { + create_test_state_with(|_| {}).await +} + +async fn create_test_state_with(tweak: impl FnOnce(&mut Config)) -> TestState { let figment = load_config_figment(None); let mut config = figment.focus("core").extract::<Config>().unwrap(); let temp_dir = TempDir::new().expect("failed to create temp dir"); config.sync.data_dir = temp_dir.path().to_string_lossy().to_string(); + // the default points at /etc/wireguard/wg0.conf, so anything that calls + // `reconfigure` would write to the host's real WireGuard config. + config.wg.config_path = temp_dir + .path() + .join("wg.conf") + .to_string_lossy() + .into_owned(); + tweak(&mut config); let options = ProxyOptions { config, - my_app_id: None, + my_app_id: b"test-app-id".to_vec(), tls_config: TlsConfig { certs: "".to_string(), key: "".to_string(), @@ -52,6 +68,48 @@ async fn test_empty_config() { insta::assert_snapshot!(wg_config); } +/// The rendered config contains the interface's WireGuard private key, so the +/// file must not be readable by anyone else. It used to be written with the +/// default mode, landing at `0o666 & !umask`. +#[cfg(unix)] +#[tokio::test] +async fn wg_config_is_written_owner_only() { + use std::os::unix::fs::PermissionsExt; + + let state = create_test_state().await; + let path = state.lock().config.wg.config_path.clone(); + + // `reconfigure` also runs `wg syncconf`, which fails without a real + // interface — that failure is logged rather than propagated, so the write + // is still exercised here. + state.lock().reconfigure().expect("reconfigure failed"); + + let rendered = std::fs::read_to_string(&path).expect("wg config was not written"); + assert!( + rendered.contains("PrivateKey"), + "test would be vacuous: the config carries no key" + ); + + let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; + assert_eq!( + mode, 0o600, + "wg config holds a private key, got mode {mode:o}" + ); +} + +/// Deterministic stand-in for a real WireGuard public key. +/// +/// Registration and the wg-config renderer both reject keys `wg` itself would +/// refuse, so test fixtures have to be 32 base64-encoded bytes like the real +/// thing. +fn test_pubkey(label: &str) -> String { + let mut key = [0u8; 32]; + for (slot, byte) in key.iter_mut().zip(label.bytes().cycle()) { + *slot = byte; + } + base64::engine::general_purpose::STANDARD.encode(key) +} + fn policy(restrict: bool, ports: &[u16]) -> PortPolicy { PortPolicy { ports: ports @@ -62,6 +120,52 @@ fn policy(restrict: bool, ports: &[u16]) -> PortPolicy { } } +#[test] +fn test_validate_wireguard_public_key() { + assert!(crate::kv::import::validate_wg_public_key( + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + ) + .is_ok()); + assert!(crate::kv::import::validate_wg_public_key("not-a-wireguard-key").is_err()); + assert!(crate::kv::import::validate_wg_public_key("AQID").is_err()); +} + +#[tokio::test] +async fn test_invalid_wireguard_public_key_does_not_register() { + let state = create_test_state().await; + let result = state.do_register_cvm("app", "instance", "invalid", "compose", Default::default()); + + assert!(result.is_err()); + assert!(!state.lock().state.instances.contains_key("instance")); +} + +#[tokio::test] +async fn test_wireguard_public_key_cannot_be_reused_by_another_instance() { + const PUBLIC_KEY: &str = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + let state = create_test_state().await; + state + .lock() + .new_client_by_id( + "instance-1", + "app", + PUBLIC_KEY, + "compose", + Default::default(), + ) + .unwrap(); + + let result = state.do_register_cvm( + "app", + "instance-2", + PUBLIC_KEY, + "compose", + Default::default(), + ); + + assert!(result.is_err()); + assert!(!state.lock().state.instances.contains_key("instance-2")); +} + #[tokio::test] async fn test_port_policy_restrict_mode_allows_listed_only() { let state = create_test_state().await; @@ -70,9 +174,9 @@ async fn test_port_policy_restrict_mode_allows_listed_only() { .new_client_by_id( "inst-allow", "app-allow", - "pubkey-allow", + &test_pubkey("pubkey-allow"), "hash-allow", - Some(policy(true, &[8080, 9090])), + Some(policy(true, &[8080, 9090])).into(), ) .unwrap(); assert!(is_port_allowed(&state.proxy, "inst-allow", 8080).is_ok()); @@ -88,10 +192,10 @@ async fn test_port_policy_disabled_allows_all() { .new_client_by_id( "inst-open", "app-open", - "pubkey-open", + &test_pubkey("pubkey-open"), "hash-open", // restrict_mode = false, but with `ports` listed: still open. - Some(policy(false, &[8080])), + Some(policy(false, &[8080])).into(), ) .unwrap(); assert!(is_port_allowed(&state.proxy, "inst-open", 8080).is_ok()); @@ -107,9 +211,9 @@ async fn test_port_policy_unknown_fails_closed() { .new_client_by_id( "inst-legacy", "app-legacy", - "pubkey-legacy", + &test_pubkey("pubkey-legacy"), "hash-legacy", - None, + Default::default(), ) .unwrap(); assert!(is_port_allowed(&state.proxy, "inst-legacy", 8080).is_err()); @@ -132,9 +236,9 @@ async fn test_admin_override_takes_precedence() { .new_client_by_id( "inst-ovr", "app-ovr", - "pubkey-ovr", + &test_pubkey("pubkey-ovr"), "hash-ovr", - Some(policy(true, &[8080])), + Some(policy(true, &[8080])).into(), ) .unwrap(); assert!(is_port_allowed(&state.proxy, "inst-ovr", 8080).is_ok()); @@ -156,9 +260,9 @@ async fn test_admin_override_can_open_what_instance_restricts() { .new_client_by_id( "inst-lock", "app-lock", - "pubkey-lock", + &test_pubkey("pubkey-lock"), "hash-lock", - Some(policy(true, &[])), + Some(policy(true, &[])).into(), ) .unwrap(); assert!(is_port_allowed(&state.proxy, "inst-lock", 8080).is_err()); @@ -178,9 +282,9 @@ async fn test_clear_admin_override_reverts_to_instance_policy() { .new_client_by_id( "inst-revert", "app-revert", - "pubkey-revert", + &test_pubkey("pubkey-revert"), "hash-revert", - Some(policy(true, &[8080])), + Some(policy(true, &[8080])).into(), ) .unwrap(); state @@ -218,9 +322,9 @@ async fn test_admin_override_survives_compose_hash_change() { .new_client_by_id( "inst-upgrade", "app-upgrade", - "pubkey-upgrade", + &test_pubkey("pubkey-upgrade"), "hash-v1", - Some(policy(true, &[8080])), + Some(policy(true, &[8080])).into(), ) .unwrap(); state @@ -234,9 +338,9 @@ async fn test_admin_override_survives_compose_hash_change() { .new_client_by_id( "inst-upgrade", "app-upgrade", - "pubkey-upgrade", + &test_pubkey("pubkey-upgrade"), "hash-v2", - Some(policy(true, &[7070, 8080])), + Some(policy(true, &[7070, 8080])).into(), ) .unwrap(); // Admin override must still be in effect. @@ -250,17 +354,2328 @@ async fn test_config() { let state = create_test_state().await; let mut info = state .lock() - .new_client_by_id("test-id-0", "app-id-0", "test-pubkey-0", "", None) + .new_client_by_id( + "test-id-0", + "app-id-0", + &test_pubkey("test-pubkey-0"), + "", + Default::default(), + ) .unwrap(); info.reg_time = SystemTime::UNIX_EPOCH; insta::assert_debug_snapshot!(info); let mut info1 = state .lock() - .new_client_by_id("test-id-1", "app-id-1", "test-pubkey-1", "", None) + .new_client_by_id( + "test-id-1", + "app-id-1", + &test_pubkey("test-pubkey-1"), + "", + Default::default(), + ) .unwrap(); info1.reg_time = SystemTime::UNIX_EPOCH; insta::assert_debug_snapshot!(info1); let wg_config = state.lock().generate_wg_config().unwrap(); insta::assert_snapshot!(wg_config); } + +#[tokio::test] +async fn gateway_top_n_batch_007_cache_health_and_invalidation() { + let state = create_test_state().await; + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + { + let mut proxy = state.lock(); + for index in 0..4 { + proxy + .new_client_by_id( + &format!("top-instance-{index}"), + "top-app", + &test_pubkey(&format!("top-key-{index}")), + "", + Some(policy(false, &[])).into(), + ) + .unwrap(); + } + proxy.handshake_cache.set_for_test(BTreeMap::from([ + (test_pubkey("top-key-0"), now), + (test_pubkey("top-key-1"), now - 1), + (test_pubkey("top-key-2"), now - 2), + (test_pubkey("top-key-3"), now - 3600), + ])); + let selected = proxy.select_top_n_hosts("top-app").unwrap(); + let selected_ids = selected + .iter() + .map(|row| row.instance_id.as_str()) + .collect::<Vec<_>>(); + assert_eq!( + selected_ids, + vec!["top-instance-0", "top-instance-1", "top-instance-2"] + ); + assert_eq!(proxy.state.top_n.len(), 1); + + proxy.handshake_cache.set_for_test(BTreeMap::from([ + (test_pubkey("top-key-0"), now - 3600), + (test_pubkey("top-key-1"), now - 3600), + (test_pubkey("top-key-2"), now - 3600), + (test_pubkey("top-key-3"), now), + ])); + let cached = proxy.select_top_n_hosts("top-app").unwrap(); + assert_eq!( + cached + .iter() + .map(|row| row.instance_id.as_str()) + .collect::<Vec<_>>(), + selected_ids + ); + + proxy + .new_client_by_id( + "top-instance-4", + "top-app", + &test_pubkey("top-key-4"), + "", + Some(policy(false, &[])).into(), + ) + .unwrap(); + assert!(proxy.state.top_n.is_empty()); + proxy.handshake_cache.set_for_test(BTreeMap::from([ + (test_pubkey("top-key-0"), now - 3600), + (test_pubkey("top-key-1"), now - 3600), + (test_pubkey("top-key-2"), now - 3600), + (test_pubkey("top-key-3"), now), + (test_pubkey("top-key-4"), now - 1), + ])); + let refreshed = proxy.select_top_n_hosts("top-app").unwrap(); + assert_eq!(refreshed.len(), 2); + assert!(refreshed + .iter() + .any(|row| row.instance_id == "top-instance-4")); + + proxy.remove_instance("top-instance-4").unwrap(); + assert!(proxy.state.top_n.is_empty()); + let direct = proxy.select_top_n_hosts("top-instance-3").unwrap(); + assert_eq!(direct.len(), 1); + assert_eq!(direct[0].instance_id, "top-instance-3"); + assert!(proxy.select_top_n_hosts("other-app").is_err()); + } +} + +/// Register `count` instances of `app` and mark every handshake fresh, so the +/// only thing that can keep an instance out of a selection is the gate. +fn register_ready_instances(proxy: &mut ProxyState, app: &str, count: usize) { + register_instances(proxy, app, count, false); +} + +/// Register `count` instances of `app` and mark every handshake fresh. +/// +/// `health_check` mirrors what the CVM declares at registration: false is a +/// legacy image the gateway never polls, true is one that starts out `Unknown` +/// and has to be polled before it can serve. +fn register_instances(proxy: &mut ProxyState, app: &str, count: usize, health_check: bool) { + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + let mut handshakes = BTreeMap::new(); + for index in 0..count { + let id = format!("{app}-{index}"); + let key = test_pubkey(&format!("{app}-key-{index}")); + proxy + .new_client_by_id( + &id, + app, + &key, + "", + ReportedCapabilities { + port_policy: Some(policy(false, &[])), + health_check: Some(health_check), + }, + ) + .unwrap(); + handshakes.insert(key, now); + } + proxy.handshake_cache.set_for_test(handshakes); +} + +/// Feed the poller's verdict in directly, standing in for a round of RPCs. +/// +/// Builds the target from the instance's current record, i.e. the case where +/// nothing changed under the poller. `observe_from` covers the case where +/// something did. +fn observe(proxy: &mut ProxyState, instance_id: &str, state: HealthState) { + let info = proxy.state.instances[instance_id].clone(); + observe_from( + proxy, + &crate::proxy::health_check::Target { + id: info.id, + ip: info.ip, + public_key: info.public_key, + }, + state, + ); +} + +/// Feed a verdict in against a specific target, so a stale one can be tested. +fn observe_from( + proxy: &mut ProxyState, + target: &crate::proxy::health_check::Target, + state: HealthState, +) { + proxy.record_instance_health( + target, + crate::proxy::health_check::Observation { + state, + reason: String::new(), + }, + ); +} + +fn selected_ids(group: &AddressGroup) -> Vec<String> { + group + .iter() + .map(|row| row.instance_id.clone()) + .collect::<Vec<_>>() +} + +/// The gate exists so an operator can pull a misbehaving instance out of the +/// rotation *and still reach it*. Dropping it from app-id selection while +/// leaving instance-id routing untouched is the whole feature. +#[tokio::test] +async fn gating_an_instance_removes_it_from_rotation_but_keeps_it_directly_reachable() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_ready_instances(&mut proxy, "gated-app", 3); + + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("gated-app").unwrap()).len(), + 3 + ); + + proxy.set_ready("gated-app-1", false).unwrap(); + + let after = selected_ids(&proxy.select_top_n_hosts("gated-app").unwrap()); + assert_eq!(after, vec!["gated-app-0", "gated-app-2"]); + + // ... but addressing it directly still resolves, which is what makes + // debugging a quarantined instance possible. + let direct = proxy.select_top_n_hosts("gated-app-1").unwrap(); + assert_eq!(selected_ids(&direct), vec!["gated-app-1"]); +} + +/// The selection cache holds a pre-gate snapshot for up to `cache_top_n`. +/// An emergency cut-off that takes 30s to apply is not an emergency cut-off. +#[tokio::test] +async fn gating_an_instance_invalidates_the_selection_cache() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_ready_instances(&mut proxy, "cache-app", 2); + + proxy.select_top_n_hosts("cache-app").unwrap(); + assert_eq!(proxy.state.top_n.len(), 1, "selection should be cached"); + + proxy.set_ready("cache-app-0", false).unwrap(); + assert!( + proxy.state.top_n.is_empty(), + "closing the gate must drop the cached selection" + ); + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("cache-app").unwrap()), + vec!["cache-app-1"] + ); +} + +/// Mark `instance_id`'s tunnel as long dead while leaving the others alone. +fn expire_handshake(proxy: &mut ProxyState, instance_id: &str) { + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + let key = proxy.state.instances[instance_id].public_key.clone(); + let mut handshakes = proxy + .latest_handshakes(None) + .unwrap() + .into_iter() + .map(|(pk, (ts, _))| (pk, ts)) + .collect::<BTreeMap<_, _>>(); + handshakes.insert(key, now.saturating_sub(3600)); + proxy.handshake_cache.set_for_test(handshakes); +} + +/// The warning exists to tell an operator they just took the app offline. An +/// instance nobody gated but whose tunnel is dead is not cover: selection will +/// not route to it either. Counting it as cover silences the warning in the one +/// case it is for. +#[tokio::test] +async fn draining_the_last_live_instance_warns_even_with_ungated_dead_ones() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_ready_instances(&mut proxy, "half-dead", 2); + expire_handshake(&mut proxy, "half-dead-1"); + + // Nobody gated -1, so a gate-only count sees cover in it. + assert!(proxy.state.instances["half-dead-1"].is_ready()); + assert_eq!(proxy.count_eligible_instances("half-dead").unwrap(), 1); + + proxy.set_ready("half-dead-0", false).unwrap(); + let warning = proxy + .drain_warning("half-dead-0", "half-dead", true, false) + .unwrap(); + assert!( + warning.is_some_and(|message| message.contains("no instance eligible")), + "draining onto a dead instance must still warn" + ); +} + +/// The message on a failed gate write asks the operator to retry. Warning on +/// the resulting count alone makes that retry claim the instance was the last +/// ready one -- a second time, about an instance that was already gated. +#[tokio::test] +async fn re_gating_an_already_gated_instance_does_not_warn_again() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_ready_instances(&mut proxy, "twice", 1); + + proxy.set_ready("twice-0", false).unwrap(); + assert!( + proxy + .drain_warning("twice-0", "twice", true, false) + .unwrap() + .is_some(), + "closing the last gate is worth saying once" + ); + + assert!( + proxy + .drain_warning("twice-0", "twice", false, false) + .unwrap() + .is_none(), + "an instance that was already gated was not the last ready one" + ); +} + +/// Health checks are inference and may be wrong, so they fail open. The +/// operator gate is an instruction, so it does not: gating every instance +/// means the app refuses new connections rather than quietly serving them. +#[tokio::test] +async fn gating_every_instance_refuses_traffic_instead_of_failing_open() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_ready_instances(&mut proxy, "drained-app", 2); + + proxy.set_ready("drained-app-0", false).unwrap(); + proxy.set_ready("drained-app-1", false).unwrap(); + + assert!( + proxy.select_top_n_hosts("drained-app").unwrap().is_empty(), + "an operator draining every instance must be obeyed, not overridden" + ); + + proxy.set_ready("drained-app-1", true).unwrap(); + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("drained-app").unwrap()), + vec!["drained-app-1"] + ); +} + +/// Instance ids are derived from attestation and survive reboots, so a +/// re-registration is usually the same box coming back. It must not rejoin the +/// rotation on its own. +#[tokio::test] +async fn the_gate_survives_re_registration() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_ready_instances(&mut proxy, "reboot-app", 2); + proxy.set_ready("reboot-app-0", false).unwrap(); + + // Same instance id, same key: a reboot re-running registration. + proxy + .new_client_by_id( + "reboot-app-0", + "reboot-app", + &test_pubkey("reboot-app-key-0"), + "", + Some(policy(false, &[])).into(), + ) + .unwrap(); + + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("reboot-app").unwrap()), + vec!["reboot-app-1"], + "a rebooted instance must stay out of rotation until an operator says otherwise" + ); +} + +/// The re-registration above is local, so the node applying it already holds +/// the gate. The case that used to lose it is a peer: the gate rode in the +/// instance record, every node rewrites that record in full from its own +/// memory on every registration, and `gateway_checker` re-registers every 180s +/// -- so a node the gate had not reached yet published a record without it and +/// won on last-write. The record it publishes now cannot carry the gate at all. +#[tokio::test] +async fn a_re_registration_by_a_node_that_never_saw_the_gate_cannot_drop_it() { + let state = create_test_state().await; + { + let mut proxy = state.lock(); + register_ready_instances(&mut proxy, "race-app", 2); + proxy.set_ready("race-app-0", false).unwrap(); + } + + // What that peer would publish: the whole instance record, rebuilt from a + // memory copy holding no gate, with a newer reg_time so it wins. + { + let mut proxy = state.lock(); + let unaware = InstanceInfo { + ready: None, + admin_port_policy: None, + reg_time: SystemTime::now(), + ..proxy.state.instances["race-app-0"].clone() + }; + state + .kv_store + .sync_instance("race-app-0", &InstanceRecord::from(&unaware)) + .unwrap(); + // ... and this node forgets it too, so only the store can answer. + proxy.state.instances.get_mut("race-app-0").unwrap().ready = None; + } + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + let mut proxy = state.lock(); + assert_eq!( + proxy.state.instances["race-app-0"].ready, + Some(false), + "the gate has its own key; a re-registration must not be able to reach it" + ); + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("race-app").unwrap()), + vec!["race-app-1"] + ); +} + +/// The two per-instance facts that must travel by different routes, asserted +/// against one record so neither can be moved onto the other's route quietly. +/// +/// What the CVM declared is the CVM's own, so it rides in the `inst/` record +/// that any node rewrites on any registration -- lose it there and a reload +/// stops polling an app that asked to be polled, which reads as "always +/// healthy". What the operator decided is not, so it must be absent from that +/// same record -- put it back and a re-registration reopens a gate an operator +/// closed, which is what giving it a key of its own was for. `InstanceRecord` +/// no longer has a field for the gate at all, so the check is that the bytes a +/// registration publishes carry none: `LegacyOverrides` is the reader that +/// would find one. +#[tokio::test] +async fn a_published_record_carries_the_declaration_and_not_the_gate() { + let state = create_test_state().await; + { + let mut proxy = state.lock(); + register_instances(&mut proxy, "split-app", 1, true); + proxy.set_ready("split-app-0", false).unwrap(); + // A re-registration is what rewrites the whole record from memory, + // i.e. the write that had both chances to get this wrong. + proxy + .new_client_by_id( + "split-app-0", + "split-app", + &test_pubkey("split-app-key-0"), + "", + ReportedCapabilities { + port_policy: Some(policy(false, &[])), + health_check: Some(true), + }, + ) + .unwrap(); + } + + let loaded = state.kv_store.load_all_instances(); + let record = loaded + .decoded + .get("split-app-0") + .expect("the registration should have published a record"); + assert_eq!( + record.health_check, + Some(true), + "the record must carry what the CVM declared, or a restart stops polling the app \ + and reads its silence as healthy" + ); + assert!( + !state + .kv_store + .legacy_instance_overrides() + .contains_key("split-app-0"), + "the operator's gate must not appear in a record a re-registration rewrites" + ); + + // And the gate is still in force, from the key it does live under. + let mut proxy = state.lock(); + assert_eq!(proxy.state.instances["split-app-0"].ready, Some(false)); + assert!(proxy.select_top_n_hosts("split-app").unwrap().is_empty()); +} + +/// The data plane, not just the store: an override an older build left inside +/// the instance record has to be in force from the first load, before anything +/// has had a chance to move it. +#[tokio::test] +async fn an_override_left_in_the_instance_record_still_reaches_the_data_plane() { + let state = create_test_state().await; + { + let mut proxy = state.lock(); + register_ready_instances(&mut proxy, "legacy-app", 2); + } + // Rewrite the record the way the previous build wrote it: the operator + // fields inside, and no `admin/` record anywhere. + { + let proxy = state.lock(); + let existing = proxy.state.instances["legacy-app-0"].clone(); + let legacy = LegacyRecord { + app_id: existing.app_id, + ip: existing.ip, + public_key: existing.public_key, + reg_time: encode_ts(existing.reg_time), + port_policy: existing.port_policy, + port_policy_hash: existing.port_policy_hash, + admin_port_policy: Some(policy(true, &[8443])), + ready: Some(false), + }; + state + .kv_store + .persistent() + .write() + .put( + crate::kv::keys::inst("legacy-app-0"), + crate::kv::encode(&legacy).unwrap(), + ) + .unwrap(); + } + + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + let proxy = state.lock(); + let migrated = &proxy.state.instances["legacy-app-0"]; + assert_eq!(migrated.ready, Some(false)); + assert_eq!(migrated.admin_port_policy, Some(policy(true, &[8443]))); + assert_eq!( + state.kv_store.instance_gate("legacy-app-0").unwrap(), + Some(false), + "and the reload should have moved them to their own keys" + ); +} + +/// The service-level counterpart of the KV test: an operator setting the gate +/// here must not discard a port-policy override a peer set and this node has +/// not applied yet. Two overrides in one record made that a whole-value +/// conflict; two keys make them independent. +#[tokio::test] +async fn setting_the_gate_does_not_discard_a_peers_port_policy_override() { + let state = create_test_state().await; + { + let mut proxy = state.lock(); + register_ready_instances(&mut proxy, "split-app", 2); + } + // A peer's override, in the store but not yet in this node's memory. + state + .kv_store + .set_instance_port_policy_override("split-app-0", Some(policy(true, &[8443]))) + .unwrap(); + state.lock().set_ready("split-app-0", false).unwrap(); + + assert_eq!( + state.kv_store.instance_gate("split-app-0").unwrap(), + Some(false) + ); + assert_eq!( + state + .kv_store + .instance_port_policy_override("split-app-0") + .unwrap(), + Some(crate::kv::PortPolicyOverride::Set(policy(true, &[8443]))), + "the gate write must not have taken the port-policy override with it" + ); +} + +/// The two overrides have a key each, so one of them existing says nothing +/// about the other. Falling back to the instance record per instance rather +/// than per key means an operator who gates an instance mid-upgrade drops the +/// port-policy override that has not been moved across yet -- silently +/// widening the ports the app serves, which is the failure the split exists to +/// remove. +/// +/// The reload path hides this: it migrates before it reads, so the key it would +/// have fallen back for already exists by then. Startup does not -- nothing may +/// be written before the WaveKV bootstrap -- so the merge itself has to be +/// right, which is what this asserts. +#[tokio::test] +async fn one_override_key_existing_does_not_answer_for_the_other() { + let state = create_test_state().await; + // A peer gated the instance through the new key. Only that key exists. + state + .kv_store + .set_instance_gate("half-moved", false) + .unwrap(); + // The port-policy override is still where the previous build left it. + let legacy = LegacyOverrides { + ready: Some(true), + admin_port_policy: Some(policy(true, &[8443])), + }; + + let (port_policy, ready) = + read_overrides(&state.kv_store, "half-moved", Some(&legacy)).unwrap(); + assert_eq!( + ready, + Some(false), + "the gate key exists, so it answers -- not the stale copy" + ); + assert_eq!( + port_policy, + Some(policy(true, &[8443])), + "the port-policy key does not exist, so the instance record still answers" + ); +} + +/// The other half of the same rule: a key that exists and says "cleared" is an +/// answer. Reading the stale copy there would undo the clear. +#[tokio::test] +async fn a_cleared_override_is_an_answer_not_an_absence() { + let state = create_test_state().await; + state + .kv_store + .set_instance_port_policy_override("cleared", None) + .unwrap(); + let legacy = LegacyOverrides { + ready: None, + admin_port_policy: Some(policy(true, &[8443])), + }; + + let (port_policy, _) = read_overrides(&state.kv_store, "cleared", Some(&legacy)).unwrap(); + assert_eq!( + port_policy, None, + "the operator cleared it; the instance record must not put it back" + ); +} + +/// The instance record as the build before the override key wrote it. +#[derive(serde::Serialize)] +struct LegacyRecord { + app_id: String, + ip: std::net::Ipv4Addr, + public_key: String, + reg_time: u64, + port_policy: Option<crate::kv::PortPolicy>, + port_policy_hash: String, + admin_port_policy: Option<crate::kv::PortPolicy>, + ready: Option<bool>, +} + +/// The test above takes the early-return branch of `new_client_by_id`, where +/// the in-memory record is reused untouched -- so it says nothing about the +/// rebuild branch, which reconstructs the record. Dropping the operator +/// overrides there left the whole suite green, which is why this test exists. +/// +/// That branch runs when the recorded IP is no longer inside +/// `wg.client_ip_range` -- an operator renumbering the range, or a record +/// restored from a differently-configured gateway. The instance is rebuilt from +/// the previous one, and anything held only in the record has to survive that. +#[tokio::test] +async fn both_operator_overrides_survive_the_ip_rebuild_path() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_ready_instances(&mut proxy, "rebuild-app", 2); + proxy.set_ready("rebuild-app-0", false).unwrap(); + proxy + .set_admin_port_policy("rebuild-app-0", policy(true, &[8443])) + .unwrap(); + + // Put the recorded IP outside the configured range, so re-registering takes + // the rebuild branch instead of the early return. + let stale_ip: Ipv4Addr = "192.0.2.7".parse().unwrap(); + assert!( + !proxy.valid_ip(stale_ip), + "the test IP must be out of range" + ); + let old_ip = proxy.state.instances["rebuild-app-0"].ip; + proxy.state.instances.get_mut("rebuild-app-0").unwrap().ip = stale_ip; + proxy.state.allocated_addresses.remove(&old_ip); + proxy.state.allocated_addresses.insert(stale_ip); + + proxy + .new_client_by_id( + "rebuild-app-0", + "rebuild-app", + &test_pubkey("rebuild-app-key-0"), + "", + ReportedCapabilities { + port_policy: Some(policy(false, &[])), + health_check: Some(false), + }, + ) + .unwrap(); + + let rebuilt = proxy.state.instances["rebuild-app-0"].clone(); + assert!( + proxy.valid_ip(rebuilt.ip), + "the rebuild path should have reallocated an in-range IP" + ); + assert_eq!( + rebuilt.ready, + Some(false), + "the operator's traffic gate was dropped by the rebuild" + ); + assert_eq!( + rebuilt.admin_port_policy, + Some(policy(true, &[8443])), + "the operator's port-policy override was dropped by the rebuild" + ); +} + +/// `persist_instance_overrides` returns a `Result` for exactly one caller: +/// `set_ready`, whose API tells an operator the gate is in force. Swallowing +/// that error left the whole suite green, so nothing pinned the one behaviour +/// the signature exists for. +#[tokio::test] +async fn a_gate_that_could_not_be_stored_is_reported_as_such() { + let state = create_test_state().await; + state + .kv_store + .fail_writes_for_test(&[crate::kv::InstanceKey::Gate]); + let mut proxy = state.lock(); + register_ready_instances(&mut proxy, "durable-app", 1); + + let err = proxy + .set_ready("durable-app-0", false) + .expect_err("a failed store write must not be reported as success"); + let message = format!("{err:#}"); + assert!( + message.contains("not durable"), + "the operator has to be told the setting did not stick: {message}" + ); + + // Still in force locally, deliberately: the operator's intent applies to + // the next connection even though it is not durable. + assert_eq!(proxy.state.instances["durable-app-0"].ready, Some(false)); +} + +/// `connect_top_n = 0` takes the random-selection path, which carries its own +/// copy of the filter. +#[tokio::test] +async fn the_random_selection_path_honours_the_gate() { + let state = create_test_state_with(|config| config.proxy.connect_top_n = 0).await; + let mut proxy = state.lock(); + register_ready_instances(&mut proxy, "random-app", 2); + proxy.set_ready("random-app-0", false).unwrap(); + + for _ in 0..16 { + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("random-app").unwrap()), + vec!["random-app-1"] + ); + } +} + +/// The gate reaches other gateways through WaveKV, not through the RPC. A node +/// that learns about it on a reload has to drop the selection it cached before +/// the change, or it keeps feeding the gated instance for up to `cache_top_n`. +#[tokio::test] +async fn a_gate_arriving_through_kv_invalidates_the_cached_selection() { + let state = create_test_state().await; + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + { + let mut proxy = state.lock(); + register_ready_instances(&mut proxy, "peer-app", 2); + proxy.select_top_n_hosts("peer-app").unwrap(); + assert_eq!(proxy.state.top_n.len(), 1, "selection should be cached"); + } + + // A peer gated peer-app-0 and the override record reached us through sync. + state + .kv_store + .set_instance_gate("peer-app-0", false) + .unwrap(); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + let mut proxy = state.lock(); + assert!( + proxy.state.top_n.is_empty(), + "a gate arriving through KV must drop the cached selection" + ); + proxy.handshake_cache.set_for_test(BTreeMap::from([ + (test_pubkey("peer-app-key-0"), now), + (test_pubkey("peer-app-key-1"), now), + ])); + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("peer-app").unwrap()), + vec!["peer-app-1"] + ); +} + +/// The reload resets health when the declared capability changes, which flips +/// eligibility. That has to drop the cached selection for the same reason a +/// gate arriving through KV does, or the stale `top_n` keeps routing to an +/// instance the reload just took out of the running. +#[tokio::test] +async fn a_capability_change_arriving_through_kv_invalidates_the_cached_selection() { + let state = create_test_state().await; + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + { + let mut proxy = state.lock(); + // Neither opted in, so both read as `Ungated`: both are eligible and + // the selection caches them. + register_instances(&mut proxy, "swap-app", 2, false); + proxy.select_top_n_hosts("swap-app").unwrap(); + assert_eq!(proxy.state.top_n.len(), 1, "selection should be cached"); + } + + // The instance came back on an image whose app does ask to be gated, and + // the record reached us through sync. It starts `Unknown` until a poll + // answers, so it is no longer eligible. + let upgraded = { + let proxy = state.lock(); + let existing = proxy.state.instances["swap-app-0"].clone(); + InstanceRecord { + app_id: existing.app_id.clone(), + ip: existing.ip, + public_key: existing.public_key.clone(), + reg_time: now, + port_policy: existing.port_policy.clone(), + port_policy_hash: existing.port_policy_hash.clone(), + health_check: Some(true), + } + }; + state + .kv_store + .sync_instance("swap-app-0", &upgraded) + .unwrap(); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + let mut proxy = state.lock(); + assert_eq!( + proxy.state.instances["swap-app-0"].health(), + HealthState::Unknown + ); + assert!( + proxy.state.top_n.is_empty(), + "a capability change arriving through KV must drop the cached selection" + ); + proxy.handshake_cache.set_for_test(BTreeMap::from([ + (test_pubkey("swap-app-key-0"), now), + (test_pubkey("swap-app-key-1"), now), + ])); + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("swap-app").unwrap()), + vec!["swap-app-1"] + ); +} + +/// Every live key in either store whose name contains `instance_id`. +/// +/// Scanned rather than listed, so a per-instance key added later without a +/// matching delete shows up here instead of leaking quietly. +fn live_keys_naming(state: &TestState, instance_id: &str) -> Vec<String> { + let mut found = Vec::new(); + for (key, entry) in state + .kv_store + .persistent() + .read() + .iter_all_including_tombstones() + { + if key.contains(instance_id) && !entry.is_deleted() { + found.push(key.clone()); + } + } + for (key, entry) in state + .kv_store + .ephemeral() + .read() + .iter_all_including_tombstones() + { + if key.contains(instance_id) && !entry.is_deleted() { + found.push(key.clone()); + } + } + found.sort(); + found +} + +/// Nothing that names a CVM may outlive it. The gate especially: instance ids +/// are recycled, so one left behind would quarantine whatever registers under +/// the id next. +/// +/// The deletes are issued unconditionally so a failure in one cannot strand the +/// others, which is only worth anything if they are all actually issued. +#[tokio::test] +async fn removing_an_instance_leaves_nothing_behind_that_names_it() { + let state = create_test_state().await; + { + let mut proxy = state.lock(); + register_ready_instances(&mut proxy, "gone-app", 2); + proxy.set_ready("gone-app-0", false).unwrap(); + proxy + .set_admin_port_policy("gone-app-0", policy(true, &[8443])) + .unwrap(); + // Seed the two ephemeral records as well, or the assertion below passes + // whether or not the deletes are issued. + state.kv_store.sync_connections("gone-app-0", 3).unwrap(); + state + .kv_store + .sync_instance_handshake("gone-app-0", 1) + .unwrap(); + } + + let node = state.kv_store.my_node_id(); + let before = live_keys_naming(&state, "gone-app-0"); + assert_eq!( + before, + vec![ + "admin/gone-app-0/port_policy".to_string(), + "admin/gone-app-0/ready".to_string(), + format!("conn/gone-app-0/{node}"), + format!("handshake/gone-app-0/{node}"), + "inst/gone-app-0".to_string(), + ], + "the fixture must seed every per-instance key, or the assertion below \ + is vacuous" + ); + + state.lock().remove_instance("gone-app-0").unwrap(); + + assert!( + live_keys_naming(&state, "gone-app-0").is_empty(), + "left behind: {:?}", + live_keys_naming(&state, "gone-app-0") + ); + // The sibling is untouched: a delete is per instance, not per app. + assert!(!live_keys_naming(&state, "gone-app-1").is_empty()); +} + +/// Selection handing over an empty group means every instance was ruled out +/// before the port policy ever looked -- an operator drained the app, or none +/// passed the handshake filter. Blaming the port policy sends whoever reads the +/// log at the wrong subsystem, and the port it names was never examined. +#[tokio::test] +async fn an_empty_candidate_group_names_the_reason_selection_had() { + let state = create_test_state().await; + { + let mut proxy = state.lock(); + register_ready_instances(&mut proxy, "drain-all", 2); + proxy.set_ready("drain-all-0", false).unwrap(); + expire_handshake(&mut proxy, "drain-all-1"); + } + let mut proxy = state.lock(); + let empty = proxy.select_top_n_hosts("drain-all").unwrap(); + assert!(empty.is_empty()); + drop(proxy); + + let err = + crate::proxy::port_policy::filter_allowed_addresses(&state.proxy, empty, "drain-all", 443) + .unwrap_err() + .to_string(); + assert!(!err.contains("port"), "the port was never checked: {err}"); + assert!( + err.contains("1 gated out by an operator"), + "the gated instance should be accounted for: {err}" + ); + assert!( + err.contains("1 with no recent WireGuard handshake"), + "the dead instance should be accounted for: {err}" + ); +} + +#[tokio::test] +async fn gating_an_unregistered_instance_is_an_error() { + let state = create_test_state().await; + let mut proxy = state.lock(); + assert!(proxy.set_ready("never-registered", false).is_err()); +} + +/// The bug this whole mechanism exists for: an instance registers during boot, +/// before its containers are up, and is immediately eligible for traffic. +#[tokio::test] +async fn a_newly_registered_instance_waits_for_its_first_health_poll() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "boot-app", 2, true); + observe(&mut proxy, "boot-app-0", HealthState::Healthy); + + // boot-app-1 has registered but not yet answered a poll, so it is not + // serving traffic on the strength of having finished a WireGuard handshake. + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("boot-app").unwrap()), + vec!["boot-app-0"] + ); + + observe(&mut proxy, "boot-app-1", HealthState::Healthy); + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("boot-app").unwrap()).len(), + 2 + ); +} + +/// An image with no `Worker.Health` is never polled and must keep serving, or +/// upgrading a fleet one instance at a time would drain the un-upgraded half. +#[tokio::test] +async fn an_agent_that_cannot_report_health_stays_eligible() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "legacy-app", 2, false); + + for index in 0..2 { + let id = format!("legacy-app-{index}"); + assert_eq!( + proxy.state.instances[&id].health(), + HealthState::Ungated, + "an agent that never declared the capability must not sit in Unknown" + ); + } + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("legacy-app").unwrap()).len(), + 2 + ); +} + +#[tokio::test] +async fn an_unhealthy_instance_drops_out_of_rotation() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "sick-app", 3, true); + for index in 0..3 { + observe( + &mut proxy, + &format!("sick-app-{index}"), + HealthState::Healthy, + ); + } + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("sick-app").unwrap()).len(), + 3 + ); + + observe(&mut proxy, "sick-app-1", HealthState::Unhealthy); + let after = selected_ids(&proxy.select_top_n_hosts("sick-app").unwrap()); + assert_eq!(after, vec!["sick-app-0", "sick-app-2"]); +} + +/// Health is a guess. A probe misconfigured across a whole app, or an agent +/// bug that lands everywhere at once, must not take the app offline. +#[tokio::test] +async fn an_app_with_no_healthy_instance_fails_open() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "open-app", 2, true); + observe(&mut proxy, "open-app-0", HealthState::Unhealthy); + observe(&mut proxy, "open-app-1", HealthState::Unhealthy); + + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("open-app").unwrap()).len(), + 2, + "every instance unhealthy should route to all of them, not to none" + ); +} + +/// The rate limiter behind the fail-open warning is keyed by app id, and an app +/// id is chosen by the tenant. Keyed on "apps this process has ever seen" +/// rather than "apps that exist", the map only ever grew: deploy, fail every +/// health check once, undeploy, repeat -- an unbounded map in a long-lived +/// process, filled by whoever can register a CVM. +#[tokio::test] +async fn the_all_unhealthy_warning_is_forgotten_with_the_app_it_is_keyed_on() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "churn-app", 2, true); + observe(&mut proxy, "churn-app-0", HealthState::Unhealthy); + observe(&mut proxy, "churn-app-1", HealthState::Unhealthy); + + // Nothing healthy left, so selecting fails open and records the warning. + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("churn-app").unwrap()).len(), + 2 + ); + assert!( + proxy.state.all_unhealthy_warned.contains_key("churn-app"), + "the fixture must have recorded a warning, or this test proves nothing" + ); + + for index in 0..2 { + proxy + .remove_instance(&format!("churn-app-{index}")) + .unwrap(); + } + + assert!( + !proxy.state.apps.contains_key("churn-app"), + "the app should be gone once its last instance is" + ); + assert!( + !proxy.state.all_unhealthy_warned.contains_key("churn-app"), + "the warning entry outlived the app it is keyed on" + ); +} + +/// Fail-open applies to health only. The operator gate is an instruction and +/// still wins, even when the surviving instance is perfectly healthy. +#[tokio::test] +async fn failing_open_on_health_does_not_reopen_the_operator_gate() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "mixed-app", 2, true); + observe(&mut proxy, "mixed-app-0", HealthState::Unhealthy); + observe(&mut proxy, "mixed-app-1", HealthState::Unhealthy); + proxy.set_ready("mixed-app-0", false).unwrap(); + + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("mixed-app").unwrap()), + vec!["mixed-app-1"], + "the gated instance must stay out even when health fails open" + ); +} + +/// A health change has to beat the `cache_top_n` window, same as the gate. +#[tokio::test] +async fn a_health_change_invalidates_the_selection_cache() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "cached-app", 2, true); + observe(&mut proxy, "cached-app-0", HealthState::Healthy); + observe(&mut proxy, "cached-app-1", HealthState::Healthy); + + proxy.select_top_n_hosts("cached-app").unwrap(); + assert_eq!(proxy.state.top_n.len(), 1); + + observe(&mut proxy, "cached-app-0", HealthState::Unhealthy); + assert!(proxy.state.top_n.is_empty()); + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("cached-app").unwrap()), + vec!["cached-app-1"] + ); +} + +/// Same rule as the operator gate: an unhealthy instance is exactly the one an +/// operator needs to open a connection to. +#[tokio::test] +async fn an_unhealthy_instance_is_still_reachable_by_instance_id() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "direct-app", 2, true); + observe(&mut proxy, "direct-app-0", HealthState::Healthy); + observe(&mut proxy, "direct-app-1", HealthState::Unhealthy); + + let direct = proxy.select_top_n_hosts("direct-app-1").unwrap(); + assert_eq!(selected_ids(&direct), vec!["direct-app-1"]); +} + +/// Re-register one instance the way a CVM does, with the key it presents. +fn re_register( + proxy: &mut ProxyState, + instance_id: &str, + app: &str, + public_key: &str, + health_check: bool, +) { + proxy + .new_client_by_id( + instance_id, + app, + public_key, + "", + ReportedCapabilities { + port_policy: Some(policy(false, &[])), + health_check: Some(health_check), + }, + ) + .unwrap(); +} + +/// A boot regenerates the CVM's WireGuard key -- its key store is cached in +/// `/run`, which is tmpfs -- so a changed key is how a restart is told apart +/// from a refresh. The old verdict describes a process that no longer exists. +#[tokio::test] +async fn a_reboot_resets_health_to_unknown() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "restart-app", 2, true); + observe(&mut proxy, "restart-app-0", HealthState::Healthy); + observe(&mut proxy, "restart-app-1", HealthState::Healthy); + + let rebooted_key = test_pubkey("restart-app-key-0-second-boot"); + re_register( + &mut proxy, + "restart-app-0", + "restart-app", + &rebooted_key, + true, + ); + // The tunnel is rebuilt under the new key, so keep both handshakes fresh: + // this test is about health, not about handshake staleness. + proxy.handshake_cache.set_for_test(BTreeMap::from([ + (rebooted_key, now_secs()), + (test_pubkey("restart-app-key-1"), now_secs()), + ])); + + assert_eq!( + proxy.state.instances["restart-app-0"].health(), + HealthState::Unknown + ); + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("restart-app").unwrap()), + vec!["restart-app-1"] + ); +} + +/// The reboot reset has to reach the *cached* selection too, not just the +/// record. +/// +/// Every other writer to `instances` dropped the app's `top_n` and this one did +/// not, so the node that took the re-registration went on serving a selection +/// computed before the reboot for up to `cache_top_n` -- routing to a CVM whose +/// containers were not up, on the node most likely to be selected, while every +/// other node in the cluster invalidated correctly through the sync path. That +/// window is exactly what health gating exists to close, so the bug hid inside +/// the feature meant to prevent it. +#[tokio::test] +async fn a_reboot_invalidates_the_selection_cached_on_the_node_that_took_it() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "reboot-cache-app", 2, true); + observe(&mut proxy, "reboot-cache-app-0", HealthState::Healthy); + observe(&mut proxy, "reboot-cache-app-1", HealthState::Healthy); + + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("reboot-cache-app").unwrap()).len(), + 2 + ); + assert_eq!(proxy.state.top_n.len(), 1, "selection should be cached"); + + // A fresh WireGuard key is how a reboot is told apart from the periodic + // re-registration: the guest caches its key store in `/run`, which is + // tmpfs. + let rebooted_key = test_pubkey("reboot-cache-app-key-0-second-boot"); + re_register( + &mut proxy, + "reboot-cache-app-0", + "reboot-cache-app", + &rebooted_key, + true, + ); + proxy.handshake_cache.set_for_test(BTreeMap::from([ + (rebooted_key, now_secs()), + (test_pubkey("reboot-cache-app-key-1"), now_secs()), + ])); + + assert_eq!( + proxy.state.instances["reboot-cache-app-0"].health(), + HealthState::Unknown + ); + assert!( + proxy.state.top_n.is_empty(), + "a reboot must drop the selection computed before it" + ); + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("reboot-cache-app").unwrap()), + vec!["reboot-cache-app-1"], + "the rebooted instance was still being served from cache" + ); +} + +/// Registration is not boot-only: the gateway-checker re-registers every 180s +/// and whenever the handshake goes stale, reusing the key cached in `/run`. +/// Resetting on those would take a healthy instance out of the rotation every +/// three minutes for no reason. +#[tokio::test] +async fn a_periodic_re_registration_keeps_the_last_health_verdict() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "refresh-app", 2, true); + observe(&mut proxy, "refresh-app-0", HealthState::Healthy); + observe(&mut proxy, "refresh-app-1", HealthState::Healthy); + + re_register( + &mut proxy, + "refresh-app-0", + "refresh-app", + &test_pubkey("refresh-app-key-0"), + true, + ); + + assert_eq!( + proxy.state.instances["refresh-app-0"].health(), + HealthState::Healthy + ); + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("refresh-app").unwrap()), + vec!["refresh-app-0", "refresh-app-1"] + ); +} + +/// The property that makes it safe not to persist health across a restart. +/// +/// A restarted process rebuilds every instance at `Unknown`, which is not +/// healthy. If the fail-open in `retain_healthy` ever stopped covering "no +/// candidate is healthy *yet*", a gateway restart would blackhole every gated +/// app in the fleet until its first round of polls answered -- and the poll +/// interval is the whole outage. Nothing else stands between the restart and +/// that, so it is pinned here rather than left to the general fail-open test, +/// which reaches the same branch from verdicts that were actually observed. +#[tokio::test] +async fn a_fleet_that_has_not_been_polled_yet_is_still_routed_to() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "restarted-app", 3, true); + + for index in 0..3 { + assert_eq!( + proxy.state.instances[&format!("restarted-app-{index}")].health(), + HealthState::Unknown, + "the fixture must be un-polled, or this test proves nothing" + ); + } + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("restarted-app").unwrap()).len(), + 3, + "a fleet that has not been polled yet must be routed to, not held out" + ); +} + +/// The reboot reset is applied by whichever node took the re-registration. +/// Every other node learns about the new boot through sync, and must not carry +/// its own pre-reboot verdict across -- otherwise the guard in +/// `record_instance_health` closes a two-second window while this leaves a +/// whole-poll-interval one open on every peer. +#[tokio::test] +async fn a_reboot_arriving_through_kv_resets_health_on_this_node_too() { + let state = create_test_state().await; + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + { + let mut proxy = state.lock(); + register_instances(&mut proxy, "peer-reboot-app", 1, true); + observe(&mut proxy, "peer-reboot-app-0", HealthState::Healthy); + } + + // The CVM rebooted and re-registered somewhere else, so it comes back with + // a fresh WireGuard key. Everything else about the record is unchanged. + let rebooted = { + let proxy = state.lock(); + let existing = proxy.state.instances["peer-reboot-app-0"].clone(); + InstanceRecord { + app_id: existing.app_id.clone(), + ip: existing.ip, + public_key: test_pubkey("peer-reboot-app-key-0-second-boot"), + reg_time: now, + port_policy: existing.port_policy.clone(), + port_policy_hash: existing.port_policy_hash.clone(), + health_check: Some(true), + } + }; + state + .kv_store + .sync_instance("peer-reboot-app-0", &rebooted) + .unwrap(); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + let proxy = state.lock(); + assert_eq!( + proxy.state.instances["peer-reboot-app-0"].health(), + HealthState::Unknown, + "the pre-reboot verdict was carried onto the new boot" + ); +} + +/// A build predating the capability field reads the record fine and rewrites it +/// without the field -- which needs no cluster, a single node rolled back and +/// forward again is enough. That record says nothing about the app's intent, so +/// reading it as "opted out" would reset health and drop the app's cached +/// selection on a record that never said so. +#[tokio::test] +async fn a_record_written_without_the_capability_field_does_not_clear_it() { + let state = create_test_state().await; + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + { + let mut proxy = state.lock(); + register_instances(&mut proxy, "mixed-app", 1, true); + observe(&mut proxy, "mixed-app-0", HealthState::Healthy); + } + + let rewritten = { + let proxy = state.lock(); + let existing = proxy.state.instances["mixed-app-0"].clone(); + InstanceRecord { + app_id: existing.app_id.clone(), + ip: existing.ip, + public_key: existing.public_key.clone(), + reg_time: now, + port_policy: existing.port_policy.clone(), + port_policy_hash: existing.port_policy_hash.clone(), + // The older build does not know the field exists. + health_check: None, + } + }; + state + .kv_store + .sync_instance("mixed-app-0", &rewritten) + .unwrap(); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + let proxy = state.lock(); + assert!( + proxy.state.instances["mixed-app-0"].health_check(), + "an absent field must not read as an opt-out" + ); + assert_eq!( + proxy.state.instances["mixed-app-0"].health(), + HealthState::Healthy, + "and must not reset the verdict this node observed" + ); +} + +/// A CVM that is powered off sits in `state.instances` for up to +/// `recycle.timeout` -- ten hours by default -- while already being excluded +/// from routing by handshake age. Polling it every round would spend a full +/// timeout slot on a machine that is gone, and enough of them stretch the round +/// past the interval and slow detection down for every live instance. +#[tokio::test] +async fn an_instance_whose_handshake_went_stale_is_not_polled() { + let state = create_test_state().await; + { + let mut proxy = state.lock(); + register_instances(&mut proxy, "stale-app", 2, true); + let stale = now_secs() - proxy.config.proxy.timeouts.handshake_stale.as_secs() - 60; + proxy.handshake_cache.set_for_test(BTreeMap::from([ + (test_pubkey("stale-app-key-0"), now_secs()), + (test_pubkey("stale-app-key-1"), stale), + ])); + } + + let targets = crate::proxy::health_check::select_targets(&state.proxy); + let ids = targets.iter().map(|t| t.id.clone()).collect::<Vec<_>>(); + assert_eq!(ids, vec!["stale-app-0"]); +} + +/// A CVM that just registered has not completed a handshake yet. Absent is not +/// the same as stale, and reading it that way would mean an instance is never +/// polled until it has already carried traffic. +#[tokio::test] +async fn an_instance_with_no_handshake_yet_is_still_polled() { + let state = create_test_state().await; + { + let mut proxy = state.lock(); + register_instances(&mut proxy, "fresh-app", 1, true); + proxy.handshake_cache.set_for_test(BTreeMap::new()); + } + + let targets = crate::proxy::health_check::select_targets(&state.proxy); + assert_eq!( + targets.len(), + 1, + "a CVM with no handshake yet must be polled" + ); +} + +/// An app that never opted in cannot be gated on the answer, so asking would +/// only produce failures to misread. +#[tokio::test] +async fn an_app_that_did_not_opt_in_is_not_polled() { + let state = create_test_state().await; + { + let mut proxy = state.lock(); + register_instances(&mut proxy, "opt-out-app", 2, false); + } + + assert!(crate::proxy::health_check::select_targets(&state.proxy).is_empty()); +} + +/// A round takes as long as its slowest instance, and a CVM can reboot inside +/// that window. The verdict polled from the previous boot must not land on the +/// new one -- it would let an instance whose containers are not up serve +/// immediately, which is the whole thing `Unknown` exists to prevent. +#[tokio::test] +async fn a_verdict_polled_before_a_reboot_is_not_applied_after_it() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "race-app", 1, true); + + // What the poller snapshotted at the start of the round. + let polled = crate::proxy::health_check::Target { + id: "race-app-0".to_string(), + ip: proxy.state.instances["race-app-0"].ip, + public_key: proxy.state.instances["race-app-0"].public_key.clone(), + }; + + // The CVM reboots and re-registers mid-round. + let rebooted_key = test_pubkey("race-app-key-0-second-boot"); + re_register(&mut proxy, "race-app-0", "race-app", &rebooted_key, true); + assert_eq!( + proxy.state.instances["race-app-0"].health(), + HealthState::Unknown + ); + + // The in-flight poll finally lands. + observe_from(&mut proxy, &polled, HealthState::Healthy); + + assert_eq!( + proxy.state.instances["race-app-0"].health(), + HealthState::Unknown, + "the previous boot's verdict was applied to the new one" + ); +} + +/// The IP can be reallocated to a different instance entirely, so it is checked +/// for the same reason the key is. +#[tokio::test] +async fn a_verdict_polled_from_a_stale_address_is_not_applied() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "addr-app", 1, true); + + let polled = crate::proxy::health_check::Target { + id: "addr-app-0".to_string(), + ip: "10.99.99.99".parse().unwrap(), + public_key: proxy.state.instances["addr-app-0"].public_key.clone(), + }; + + observe_from(&mut proxy, &polled, HealthState::Healthy); + assert_eq!( + proxy.state.instances["addr-app-0"].health(), + HealthState::Unknown, + "a verdict polled from an address this instance no longer has was applied" + ); +} + +/// The debug registration path has no CVM to ask, so it reports nothing about +/// the capability. It must not be able to take a real instance out of polling +/// for good by re-registering its id. +#[tokio::test] +async fn a_registration_that_reports_no_capability_leaves_it_alone() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "debug-app", 1, true); + observe(&mut proxy, "debug-app-0", HealthState::Healthy); + + proxy + .new_client_by_id( + "debug-app-0", + "debug-app", + &test_pubkey("debug-app-key-0"), + "", + ReportedCapabilities { + port_policy: Some(policy(false, &[])), + health_check: None, + }, + ) + .unwrap(); + + assert!( + proxy.state.instances["debug-app-0"].health_check(), + "a caller with nothing to say must not clear the declaration" + ); + assert_eq!( + proxy.state.instances["debug-app-0"].health(), + HealthState::Healthy, + "and must not reset the verdict either" + ); +} + +/// A different capability means a different image, so the verdict belongs to +/// something that is no longer running even if the key survived. +#[tokio::test] +async fn a_changed_health_capability_resets_health() { + let state = create_test_state().await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "swap-app", 1, true); + observe(&mut proxy, "swap-app-0", HealthState::Healthy); + + re_register( + &mut proxy, + "swap-app-0", + "swap-app", + &test_pubkey("swap-app-key-0"), + false, + ); + + assert_eq!( + proxy.state.instances["swap-app-0"].health(), + HealthState::Ungated + ); +} + +/// Turning polling off must restore the old behaviour exactly. Nothing ever +/// leaves `Unknown` once the poller is not running, and in a fleet that also +/// has apps which never opted in -- `Ungated`, which counts as healthy -- the +/// filter would otherwise drop every opted-in instance, forever. +#[tokio::test] +async fn disabling_polling_keeps_both_old_and_new_instances_eligible() { + let state = create_test_state_with(|config| config.proxy.health_check.enabled = false).await; + let mut proxy = state.lock(); + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + + // One legacy instance and one that declares the endpoint but, with the + // poller off, will never answer. + let mut handshakes = BTreeMap::new(); + for (index, has_endpoint) in [false, true].into_iter().enumerate() { + let id = format!("off-app-{index}"); + let key = test_pubkey(&format!("off-app-key-{index}")); + proxy + .new_client_by_id( + &id, + "off-app", + &key, + "", + ReportedCapabilities { + port_policy: Some(policy(false, &[])), + health_check: Some(has_endpoint), + }, + ) + .unwrap(); + handshakes.insert(key, now); + } + proxy.handshake_cache.set_for_test(handshakes); + + assert_eq!( + proxy.state.instances["off-app-1"].health(), + HealthState::Unknown, + "the state is still tracked; it just must not be allowed to gate" + ); + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("off-app").unwrap()).len(), + 2, + "with polling disabled every instance stays eligible" + ); +} + +/// Health is this node's own observation and WaveKV does not carry it. A sync +/// round must not reset it, or the whole fleet drops to `Unknown` every time a +/// peer writes anything. +#[tokio::test] +async fn a_kv_reload_does_not_reset_this_node_s_health_observations() { + let state = create_test_state().await; + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + { + let mut proxy = state.lock(); + register_instances(&mut proxy, "sync-app", 2, true); + observe(&mut proxy, "sync-app-0", HealthState::Healthy); + observe(&mut proxy, "sync-app-1", HealthState::Unhealthy); + } + + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + let mut proxy = state.lock(); + assert_eq!( + proxy.state.instances["sync-app-0"].health(), + HealthState::Healthy + ); + assert_eq!( + proxy.state.instances["sync-app-1"].health(), + HealthState::Unhealthy + ); + proxy.handshake_cache.set_for_test(BTreeMap::from([ + (test_pubkey("sync-app-key-0"), now), + (test_pubkey("sync-app-key-1"), now), + ])); + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("sync-app").unwrap()), + vec!["sync-app-0"] + ); +} + +/// The two selection paths share `retain_healthy` but each decides for itself +/// whether health is allowed to gate, so a refactor could easily fix one and +/// miss the other. This pins the combination the other disabled-polling test +/// does not reach. +/// +/// The fleet is deliberately mixed. With two `Unknown` instances the fail-open +/// branch keeps both regardless, so the switch is never reached and the test +/// passes whether or not the code under test still exists -- which is exactly +/// what it did before, verified by deleting the check and watching every test +/// still pass. One healthy peer makes the retain find something to keep and +/// forces the filter to actually run. +#[tokio::test] +async fn disabling_polling_also_keeps_the_random_path_open() { + let state = create_test_state_with(|config| { + config.proxy.health_check.enabled = false; + config.proxy.connect_top_n = 0; + }) + .await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "off-rand-app", 2, true); + observe(&mut proxy, "off-rand-app-0", HealthState::Healthy); + + // One is healthy and one is `Unknown`, so with polling on the `Unknown` one + // would be filtered out rather than saved by fail-open. With polling off + // both must stay selectable. + assert_eq!( + proxy.state.instances["off-rand-app-0"].health(), + HealthState::Healthy + ); + assert_eq!( + proxy.state.instances["off-rand-app-1"].health(), + HealthState::Unknown + ); + + let mut seen = std::collections::BTreeSet::new(); + for _ in 0..32 { + let picked = selected_ids(&proxy.select_top_n_hosts("off-rand-app").unwrap()); + assert_eq!(picked.len(), 1, "the random path returns a single host"); + seen.insert(picked[0].clone()); + } + assert_eq!( + seen.len(), + 2, + "both instances must remain reachable, got {seen:?}" + ); +} + +/// `connect_top_n = 0` takes the random path, which filters separately. +#[tokio::test] +async fn the_random_selection_path_honours_health_and_fails_open() { + let state = create_test_state_with(|config| config.proxy.connect_top_n = 0).await; + let mut proxy = state.lock(); + register_instances(&mut proxy, "rand-app", 2, true); + observe(&mut proxy, "rand-app-0", HealthState::Unhealthy); + observe(&mut proxy, "rand-app-1", HealthState::Healthy); + + for _ in 0..16 { + assert_eq!( + selected_ids(&proxy.select_top_n_hosts("rand-app").unwrap()), + vec!["rand-app-1"] + ); + } + + // With nothing healthy left, the random path must fail open too. + observe(&mut proxy, "rand-app-1", HealthState::Unhealthy); + for _ in 0..16 { + assert_eq!( + proxy.select_top_n_hosts("rand-app").unwrap().len(), + 1, + "fail-open should still yield a host on the random path" + ); + } +} + +/// Write a record straight into the KV store, bypassing registration, the way +/// a peer's sync round would. +fn sync_from_peer(state: &TestState, instance_id: &str, ip: &str, public_key: &str) { + sync_from_peer_at(state, instance_id, ip, public_key, 1); +} + +fn sync_from_peer_at( + state: &TestState, + instance_id: &str, + ip: &str, + public_key: &str, + reg_time: u64, +) { + state + .kv_store + .sync_instance( + instance_id, + &InstanceRecord { + app_id: "peer-app".to_string(), + ip: ip.parse().unwrap(), + public_key: public_key.to_string(), + reg_time, + port_policy: None, + port_policy_hash: String::new(), + health_check: Some(false), + }, + ) + .unwrap(); +} + +#[tokio::test] +async fn a_poisoned_peer_record_costs_only_its_own_instance() { + let state = create_test_state().await; + sync_from_peer(&state, "good", "10.0.0.41", &test_pubkey("good-key")); + // A key `wg` refuses makes `wg syncconf` reject the entire config file, so + // this record must never reach ProxyState or the rendered peer list. + sync_from_peer( + &state, + "poisoned", + "10.0.0.42", + "not-a-key\nEndpoint = 1.2.3.4:1", + ); + // The gateway's own wg address, claimed by an instance. + sync_from_peer( + &state, + "steals-gateway-ip", + "10.0.0.1", + &test_pubkey("other"), + ); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + let proxy = state.lock(); + assert!(proxy.state.instances.contains_key("good")); + assert!(!proxy.state.instances.contains_key("poisoned")); + assert!(!proxy.state.instances.contains_key("steals-gateway-ip")); + + let rendered = proxy.generate_wg_config().unwrap(); + assert!(rendered.contains(&test_pubkey("good-key"))); + assert!(!rendered.contains("Endpoint = 1.2.3.4:1")); +} + +#[tokio::test] +async fn a_cvm_registered_on_another_node_becomes_a_wg_peer_here() { + let state = create_test_state().await; + // What a peer node allocated out of its own slice. The `test-run` suites + // give each node a /24 of its own, so a peer's address + // is outside this node's pool *and* outside its interface network — yet + // every CVM is handed every gateway as a WireGuard server, so this node + // still has to carry it. + let peer_ip = "10.0.42.5"; + assert!(!state.config.wg.is_valid_client_ip(peer_ip.parse().unwrap())); + sync_from_peer(&state, "peer-node-cvm", peer_ip, &test_pubkey("far")); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + let proxy = state.lock(); + assert!( + proxy.state.instances.contains_key("peer-node-cvm"), + "refusing a peer node's instance leaves each gateway serving only its own CVMs" + ); + let rendered = proxy.generate_wg_config().unwrap(); + assert!(rendered.contains(peer_ip), "peer missing from wg.conf"); +} + +#[tokio::test] +async fn an_instance_deleted_on_another_node_stops_being_routable_here() { + let state = create_test_state().await; + sync_from_peer( + &state, + "peer-instance", + "10.0.0.40", + &test_pubkey("peer-key"), + ); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + assert!(state.lock().state.instances.contains_key("peer-instance")); + + // The remote node recycled the CVM; until the deletion is applied locally + // the deregistered instance keeps receiving proxied traffic. + state + .kv_store + .sync_delete_instance("peer-instance") + .unwrap(); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + let proxy = state.lock(); + assert!(!proxy.state.instances.contains_key("peer-instance")); + assert!(!proxy.state.apps.contains_key("peer-app")); + assert!(!proxy + .state + .allocated_addresses + .contains(&"10.0.0.40".parse().unwrap())); +} + +/// Whether this node's ephemeral observations of `instance_id` are live. +fn observations_are_live(state: &TestState, instance_id: &str) -> (bool, bool) { + let node = state.kv_store.my_node_id(); + let store = state.kv_store.ephemeral().read(); + let live = |key: String| store.get(&key).is_some_and(|entry| !entry.is_deleted()); + ( + live(crate::kv::keys::conn(instance_id, node)), + live(crate::kv::keys::handshake(instance_id, node)), + ) +} + +/// A delete can only withdraw the deleting node's own `conn/` and `handshake/` +/// keys -- those are the only ones it owns. Every other node has to withdraw +/// its own when it learns of the deletion, or its observation of a CVM that no +/// longer exists is never collected: nothing else iterates those prefixes, and +/// the recycle loop cannot see an instance it has already dropped from memory. +#[tokio::test] +async fn learning_of_a_remote_deletion_withdraws_this_nodes_observations() { + let state = create_test_state().await; + sync_from_peer( + &state, + "peer-instance", + "10.0.0.40", + &test_pubkey("peer-key"), + ); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + // This node has been watching it, as every node in the cluster does. + state.kv_store.sync_connections("peer-instance", 4).unwrap(); + state + .kv_store + .sync_instance_handshake("peer-instance", 1) + .unwrap(); + assert_eq!(observations_are_live(&state, "peer-instance"), (true, true)); + + // The tombstone arrives through sync: the peer withdrew its own keys, not + // ours. Written directly rather than through `sync_delete_instance`, which + // is what the *deleting* node calls. + state + .kv_store + .persistent() + .write() + .delete(crate::kv::keys::inst("peer-instance")) + .unwrap(); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + assert!(!state.lock().state.instances.contains_key("peer-instance")); + assert_eq!( + observations_are_live(&state, "peer-instance"), + (false, false), + "this node's observations must not outlive the instance they describe" + ); +} + +#[tokio::test] +async fn proxy_state_adopts_the_wavekv_winner_regardless_of_value_reg_time() { + let state = create_test_state().await; + sync_from_peer_at( + &state, + "contended", + "10.0.0.40", + &test_pubkey("old-key"), + 300, + ); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + state + .lock() + .state + .instances + .get("contended") + .unwrap() + .connections + .store(7, Ordering::Relaxed); + + // This is the value WaveKV selected using its own entry metadata. Its + // payload timestamp is older, so comparing reg_time again would leave the + // data plane permanently materializing a losing value. + sync_from_peer_at( + &state, + "contended", + "10.0.0.41", + &test_pubkey("winner-key"), + 100, + ); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + let proxy = state.lock(); + let instance = &proxy.state.instances["contended"]; + assert_eq!(instance.ip, "10.0.0.41".parse::<Ipv4Addr>().unwrap()); + assert_eq!(instance.public_key, test_pubkey("winner-key")); + assert_eq!(encode_ts(instance.reg_time), 100); + assert_eq!(instance.num_connections(), 7); + assert!(!proxy + .state + .allocated_addresses + .contains(&"10.0.0.40".parse().unwrap())); +} + +#[tokio::test] +async fn a_local_registration_survives_a_reload_that_cannot_see_it_yet() { + let state = create_test_state().await; + state + .lock() + .new_client_by_id( + "fresh", + "fresh-app", + &test_pubkey("fresh-key"), + "", + Default::default(), + ) + .unwrap(); + // Stand in for a KV write that failed: the instance exists locally only. + // Evicting it would black-hole a live CVM until it re-registers. + state + .kv_store + .persistent() + .write() + .delete(crate::kv::keys::inst("fresh")) + .unwrap(); + + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + assert!(state.lock().state.instances.contains_key("fresh")); +} + +#[tokio::test] +async fn a_record_that_stops_validating_keeps_the_instance_it_describes() { + let state = create_test_state().await; + sync_from_peer(&state, "peer-instance", "10.0.0.40", &test_pubkey("good")); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + assert!(state.lock().state.instances.contains_key("peer-instance")); + + // A peer overwrites the record with one this node refuses. "I cannot read + // your current state" is not "you were deleted": the last known-good state + // keeps the CVM reachable until a usable record or a real deletion arrives. + sync_from_peer(&state, "peer-instance", "10.0.0.40", "not-a-key"); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + let proxy = state.lock(); + let instance = proxy + .state + .instances + .get("peer-instance") + .expect("a rejected record must not evict the instance"); + assert_eq!(instance.public_key, test_pubkey("good")); + assert!(proxy.generate_wg_config().unwrap().contains("10.0.0.40")); +} + +#[tokio::test] +async fn an_undecodable_record_keeps_the_instance_it_describes() { + let state = create_test_state().await; + sync_from_peer(&state, "peer-instance", "10.0.0.40", &test_pubkey("good")); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + // A torn write, or a record written by a build whose schema this one cannot + // read. Folding that into "absent" would let a rolling upgrade evict every + // instance the newer nodes registered. + state + .kv_store + .persistent() + .write() + .put( + crate::kv::keys::inst("peer-instance"), + b"not-messagepack".to_vec(), + ) + .unwrap(); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + assert!(state.lock().state.instances.contains_key("peer-instance")); +} + +#[tokio::test] +async fn an_operator_can_remove_a_cvm_whose_kv_record_is_unreadable() { + let state = create_test_state().await; + sync_from_peer(&state, "peer-instance", "10.0.0.40", &test_pubkey("good")); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + state + .kv_store + .persistent() + .write() + .put( + crate::kv::keys::inst("peer-instance"), + b"not-messagepack".to_vec(), + ) + .unwrap(); + + let removal = state.proxy.remove_cvm("peer-instance").unwrap(); + assert!(removal.record_existed); + assert!(removal.removed_locally); + assert!(!state.lock().state.instances.contains_key("peer-instance")); + let loaded = state.kv_store.load_all_instances(); + assert!(!loaded.decoded.contains_key("peer-instance")); + assert!(!loaded.undecodable.contains_key("peer-instance")); + + // The recovery operation is safe to retry after a timeout or lost reply, + // and the retry tells the operator there was nothing left to remove. + let retry = state.proxy.remove_cvm("peer-instance").unwrap(); + assert!(!retry.record_existed); + assert!(!retry.removed_locally); +} + +/// A removal that raced a peer's, or one whose earlier attempt died between +/// the tombstone and the telemetry keys, leaves `conn/`/`handshake/` records +/// with no instance behind them. Nothing garbage-collects those, so re-issuing +/// the removal has to sweep them up rather than stop at "no record to delete". +#[tokio::test] +async fn removing_a_cvm_clears_its_telemetry_even_with_no_record_left() { + let state = create_test_state().await; + state.kv_store.sync_connections("orphan", 3).unwrap(); + state.kv_store.sync_instance_handshake("orphan", 1).unwrap(); + assert_eq!( + observations_are_live(&state, "orphan"), + (true, true), + "seeded telemetry records should be live" + ); + + // No `inst/` record exists, so the removal reports nothing was there -- + // and still deletes both telemetry records. + let removal = state.proxy.remove_cvm("orphan").unwrap(); + assert!(!removal.record_existed); + assert!(!removal.removed_locally); + let leaked = live_keys_naming(&state, "orphan"); + assert!(leaked.is_empty(), "left behind: {leaked:?}"); +} + +#[tokio::test] +async fn removing_an_unknown_cvm_reports_that_nothing_existed() { + let state = create_test_state().await; + + // A mistyped instance_id must not be mistaken for a successful removal. + let removal = state.proxy.remove_cvm("no-such-instance").unwrap(); + assert!(!removal.record_existed); + assert!(!removal.removed_locally); +} + +#[tokio::test] +async fn rejected_instance_records_are_visible_to_the_operator() { + let state = create_test_state().await; + sync_from_peer(&state, "peer-instance", "10.0.0.40", &test_pubkey("good")); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + assert!(state.proxy.rejected_instances().is_empty()); + + state + .kv_store + .persistent() + .write() + .put( + crate::kv::keys::inst("peer-instance"), + b"not-messagepack".to_vec(), + ) + .unwrap(); + + // The operator can see what is wrong — with the actual decode error, and + // whether removal would also drop live routing — without grepping logs. + let rejected = state.proxy.rejected_instances(); + assert_eq!(rejected.len(), 1); + assert_eq!(rejected[0].rejected.instance_id, "peer-instance"); + assert!(format!("{:#}", rejected[0].rejected.reason).contains("corrupt record")); + assert!(rejected[0].active_locally); + + // Once removed, the record no longer shows up as rejected. + state.proxy.remove_cvm("peer-instance").unwrap(); + assert!(state.proxy.rejected_instances().is_empty()); +} + +#[tokio::test] +async fn an_operator_can_remove_a_decommissioned_node() { + let state = create_test_state().await; + let kv = &state.kv_store; + kv.register_peer_url(7, "https://gw7.example.com:9202") + .unwrap(); + kv.sync_node( + 7, + &crate::kv::NodeData { + uuid: b"gw7-uuid".to_vec(), + url: "https://gw7.example.com:9202".to_string(), + wg_public_key: String::new(), + wg_endpoint: String::new(), + wg_ip: String::new(), + }, + ) + .unwrap(); + + let removal = state.proxy.remove_node(7).unwrap(); + assert!(removal.record_existed); + assert!(removal.removed_from_peer_set); + assert!(kv.get_peer_url(7).is_none()); + assert!( + kv.is_peer_removed(7), + "removal leaves the durable marker the sync lockout reads" + ); + assert!(!kv.load_all_nodes().contains_key(&7)); + let peers = kv.persistent().read().status().peers; + assert!(!peers.iter().any(|peer| peer.id == 7)); + + // The recovery operation is safe to retry after a timeout or lost reply, + // and the retry tells the operator there was nothing left to remove. + let retry = state.proxy.remove_node(7).unwrap(); + assert!(!retry.record_existed); + assert!(!retry.removed_from_peer_set); + + // A node known only by its sync address (registered via SetNodeUrl but + // never booted) still reports record_existed. + kv.register_peer_url(8, "https://gw8.example.com:9202") + .unwrap(); + let removal = state.proxy.remove_node(8).unwrap(); + assert!(removal.record_existed); +} + +#[tokio::test] +async fn a_node_cannot_remove_itself() { + let state = create_test_state().await; + let my_id = state.proxy.config.sync.node_id; + assert!(state.proxy.remove_node(my_id).is_err()); +} + +#[tokio::test] +async fn peers_prune_nodes_removed_on_another_gateway() { + let state = create_test_state().await; + let kv = &state.kv_store; + + // Simulate observing a removal performed elsewhere: the __peer_addr + // tombstone arrives via replication, not through this node's admin API. + kv.register_peer_url(9, "https://gw9.example.com:9202") + .unwrap(); + kv.persistent() + .write() + .delete(crate::kv::keys::peer_addr(9)) + .unwrap(); + kv.prune_removed_peers(); + let peers = kv.persistent().read().status().peers; + assert!(!peers.iter().any(|peer| peer.id == 9)); + + // A peer whose address record was never written is not dropped: + // bootstrap can add a peer before its address has synced in. + kv.add_peer(11).unwrap(); + kv.prune_removed_peers(); + let peers = kv.persistent().read().status().peers; + assert!(peers.iter().any(|peer| peer.id == 11)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn remove_node_reports_peer_membership_despite_a_racing_watcher() { + let state = create_test_state().await; + let kv = state.kv_store.clone(); + + // The production watch tasks prune the peer set as soon as either signal + // lands: the __peer_addr tombstone wakes the address watcher, and the + // removal marker -- written first -- wakes the marker watcher. + // remove_node must capture membership before writing anything, or the + // answer it returns would depend on which of the racers gets there + // first. Both watchers are driven here at full speed; the marker one is + // the sharper race, because the marker is remove_node's very first + // write. + let mut addr_rx = kv.watch_peer_addrs(); + let kv_for_addrs = kv.clone(); + let addr_watcher = tokio::spawn(async move { + while addr_rx.changed().await.is_ok() { + kv_for_addrs.prune_removed_peers(); + } + }); + let mut marker_rx = kv.watch_peer_removed(); + let kv_for_markers = kv.clone(); + let marker_watcher = tokio::spawn(async move { + while marker_rx.changed().await.is_ok() { + kv_for_markers.prune_removed_peers(); + } + }); + + for node_id in 100..120 { + kv.register_peer_url(node_id, "https://gw.example.com:9202") + .unwrap(); + let removal = state.proxy.remove_node(node_id).unwrap(); + assert!(removal.record_existed); + assert!( + removal.removed_from_peer_set, + "node {node_id}: membership must be captured before the first write lands" + ); + let peers = kv.persistent().read().status().peers; + assert!(!peers.iter().any(|peer| peer.id == node_id)); + } + + addr_watcher.abort(); + marker_watcher.abort(); +} + +#[tokio::test] +async fn a_zt_domain_with_a_corrupt_config_can_still_be_deleted() { + let state = create_test_state().await; + let kv = &state.kv_store; + + kv.persistent() + .write() + .put( + crate::kv::keys::zt_domain_config("bad.example.com"), + b"not-messagepack".to_vec(), + ) + .unwrap(); + + // The corrupt record is invisible to reads, but must still be deletable — + // otherwise it would be permanently stuck. + assert!(kv.get_zt_domain_config("bad.example.com").is_none()); + assert!(kv.zt_domain_config_exists("bad.example.com")); + + kv.delete_zt_domain_config("bad.example.com").unwrap(); + assert!(!kv.zt_domain_config_exists("bad.example.com")); +} + +#[tokio::test] +async fn an_instance_that_lost_an_ip_conflict_stops_being_routable() { + let state = create_test_state().await; + sync_from_peer_at(&state, "loser", "10.0.0.40", &test_pubkey("loser"), 300); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + assert!(state.lock().state.instances.contains_key("loser")); + + // An older registration claims the same IP. Unlike an unreadable record, + // this one is readable and says the address belongs to someone else, so + // the loser has to go: two peers on one IP is a config `wg` will not load. + sync_from_peer_at(&state, "winner", "10.0.0.40", &test_pubkey("winner"), 100); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + let proxy = state.lock(); + assert!(!proxy.state.instances.contains_key("loser")); + assert!(proxy.state.instances.contains_key("winner")); +} + +#[tokio::test] +async fn a_future_dated_registration_cannot_park_itself_in_the_data_plane() { + let state = create_test_state().await; + // Both the removal pass and `recycle()` age instances with + // `elapsed().unwrap_or_default()`, which reads a future `reg_time` as zero + // age — an instance dated forward would survive deletion and recycling + // alike until this process restarts. + let far_future = now_secs() + 365 * 24 * 3600; + sync_from_peer_at(&state, "zombie", "10.0.0.40", &test_pubkey("z"), far_future); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + + assert!(!state.lock().state.instances.contains_key("zombie")); +} + +#[tokio::test] +async fn a_stuck_bad_record_is_reported_once_and_again_when_it_recovers() { + let state = create_test_state().await; + sync_from_peer(&state, "peer-instance", "10.0.0.40", "not-a-key"); + + // A record is refused for what it contains, so it stays refused until + // someone rewrites it. `reported_rejections` is what keeps the reload from + // re-emitting the same `error!` on every round: a reason already in the map + // is not logged again, so a second identical reload must leave it untouched. + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + let first = state.lock().reported_rejections.clone(); + assert_eq!(first.len(), 1); + assert!(first["peer-instance"].contains("public key")); + + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + assert_eq!(state.lock().reported_rejections, first); + + // Recovery is a transition too, and clearing the entry is what lets a + // later relapse be reported instead of silently swallowed. + sync_from_peer(&state, "peer-instance", "10.0.0.40", &test_pubkey("good")); + reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap(); + assert!(state.lock().reported_rejections.is_empty()); +} + +/// The GC trigger is a pure function of the replicated write count and the +/// shared pace, so every node in a cluster crosses each collection boundary +/// together without reading a clock. The digest covers tombstones, so a node +/// that has collected and one that has not look diverged to wavekv's detector, +/// which forces a full re-exchange; nodes collecting on their own phase would +/// pay that every cycle. +#[test] +fn tombstone_collection_triggers_on_write_count_boundaries_not_on_time() { + let w = |persistent, ephemeral| ReplicatedWrites { + persistent, + ephemeral, + }; + + // Never collected: due regardless of the count — the backlog case, where a + // store that never writes again would otherwise never shed the tombstones + // it already holds. + assert!(tombstone_collection_due(None, w(0, 0), 100)); + + // Inside the boundary nothing is due, no matter how much time has passed. + assert!(!tombstone_collection_due(Some(w(50, 0)), w(99, 0), 100)); + // Crossing it is what makes collection due, on either store. + assert!(tombstone_collection_due(Some(w(50, 0)), w(100, 0), 100)); + assert!(tombstone_collection_due(Some(w(50, 0)), w(50, 100), 100)); + + // Two nodes whose views of the same state are a beat apart agree on every + // boundary: whichever sees the crossing later still sees it. + let earlier = w(99, 0); + let later = w(101, 0); + assert!(!tombstone_collection_due(Some(w(50, 0)), earlier, 100)); + assert!(tombstone_collection_due(Some(w(50, 0)), later, 100)); + + // Digest repair can lower an ack; a count that stepped back is not due + // until it crosses the next boundary again. + assert!(!tombstone_collection_due(Some(w(100, 0)), w(90, 0), 100)); +} diff --git a/dstack/gateway/src/metrics.rs b/dstack/gateway/src/metrics.rs new file mode 100644 index 000000000..bf0421902 --- /dev/null +++ b/dstack/gateway/src/metrics.rs @@ -0,0 +1,728 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Prometheus metrics for the gateway. +//! +//! Counters live in process-wide atomics rather than on `Proxy` because the +//! sites that need them -- the KV codec, `reconfigure()` -- run on code paths +//! that hold no handle to the proxy state. `proxy::NUM_CONNECTIONS` and +//! `proxy::stats` already work this way. +//! +//! Gauges are not stored: they are sampled from live state when a scrape +//! arrives, so a scrape never has to be kept in sync with a mutation. +//! +//! Label values that come from replicated state (domains, and therefore +//! anything a peer can name) are escaped, and the decode-failure label set is +//! a fixed list of known prefixes plus `other`, so a peer cannot inflate +//! cardinality by inventing keys. + +use std::cell::Cell; +use std::fmt::Write as _; +use std::sync::atomic::{AtomicU64, Ordering}; + +use dstack_gateway_rpc::ProxyAccelStatus; + +use crate::kv::keys; + +/// Key prefixes that get their own decode-failure series. +/// +/// Taken from `kv::keys` rather than spelled out here: a prefix that is +/// renamed there and not here would not break anything loudly, it would just +/// start counting that key space under `other`. +/// +/// Anything unmatched lands in `other`, which is what keeps a peer from +/// inventing key spaces to inflate cardinality. +const METERED_PREFIXES: [&str; 9] = [ + keys::INST_PREFIX, + keys::NODE_PREFIX, + keys::CONN_PREFIX, + keys::HANDSHAKE_PREFIX, + keys::LAST_SEEN_NODE_PREFIX, + keys::PEER_ADDR_PREFIX, + keys::CERT_PREFIX, + keys::DNS_CRED_PREFIX, + keys::GLOBAL_PREFIX, +]; + +const OTHER_PREFIX: &str = "other"; + +/// Ceiling on `cert_not_after` series, far above any real deployment. +const MAX_CERT_SERIES: usize = 256; + +static DECODE_FAILURES: [AtomicU64; METERED_PREFIXES.len() + 1] = + [const { AtomicU64::new(0) }; METERED_PREFIXES.len() + 1]; +static WG_RECONFIGURE_TOTAL: AtomicU64 = AtomicU64::new(0); +static WG_RECONFIGURE_FAILURES: AtomicU64 = AtomicU64::new(0); +static KV_PERSIST_FAILURES: AtomicU64 = AtomicU64::new(0); +static KV_WAL_SYNC_FAILURES: AtomicU64 = AtomicU64::new(0); +static SYNC_REJECTED: AtomicU64 = AtomicU64::new(0); + +thread_local! { + /// Set while a scrape is sampling live state. + /// + /// A scrape reads the same replicated records the data path reads, through + /// the same decoding helpers, so without this it would feed the very + /// counter it is about to report: one permanently corrupt record would + /// increment `decode_failures` once per scrape forever, and `rate()` over + /// it would measure the scrape interval rather than anything about the + /// store. Observing must not be indistinguishable from failing. + static SAMPLING: Cell<bool> = const { Cell::new(false) }; +} + +/// Suppresses decode-failure counting until dropped. +/// +/// Correctness depends on the sampler staying synchronous: it must not yield +/// to the runtime while this is alive, or the flag would apply to whatever +/// else the runtime schedules onto this thread. +pub(crate) struct ScrapeGuard(bool); + +impl Drop for ScrapeGuard { + fn drop(&mut self) { + SAMPLING.with(|sampling| sampling.set(self.0)); + } +} + +/// Mark the current thread as sampling for a scrape. See [`ScrapeGuard`]. +#[must_use = "decode-failure suppression ends as soon as the guard is dropped"] +pub(crate) fn scrape_guard() -> ScrapeGuard { + ScrapeGuard(SAMPLING.with(|sampling| sampling.replace(true))) +} + +/// Record that a replicated value could not be decoded. +/// +/// A decode failure makes the record invisible to the data plane with nothing +/// but a log line to say so, which is how a single corrupt record turns into +/// "that CVM silently stopped being routable". +/// +/// Counting is suppressed while a scrape samples; see [`ScrapeGuard`]. +pub(crate) fn record_decode_failure(key: &str) { + if SAMPLING.with(Cell::get) { + return; + } + DECODE_FAILURES[prefix_index(key)].fetch_add(1, Ordering::Relaxed); +} + +/// Record the outcome of pushing a new WireGuard config. +/// +/// Covers the whole of `reconfigure()`, not just `wg syncconf`: rendering and +/// writing the config can fail too, and all three leave the data plane on its +/// previous routing table while the gateway keeps answering. `wg syncconf` +/// additionally rejects the *whole* file when one peer stanza is bad, and its +/// call site can only log that, so without a counter a gateway that stopped +/// applying routing updates looks healthy. +pub(crate) fn record_wg_reconfigure(ok: bool) { + WG_RECONFIGURE_TOTAL.fetch_add(1, Ordering::Relaxed); + if !ok { + WG_RECONFIGURE_FAILURES.fetch_add(1, Ordering::Relaxed); + } +} + +/// Record a failed periodic snapshot. Repeated failures mean the node is one +/// restart away from replaying a very long WAL, or from losing the writes it +/// never managed to snapshot. +pub(crate) fn record_kv_persist_failure() { + KV_PERSIST_FAILURES.fetch_add(1, Ordering::Relaxed); +} + +pub(crate) fn record_kv_wal_sync_failure() { + KV_WAL_SYNC_FAILURES.fetch_add(1, Ordering::Relaxed); +} + +/// Record a sync envelope of ours that a peer rejected with HTTP 403. +pub(crate) fn record_sync_rejected() { + SYNC_REJECTED.fetch_add(1, Ordering::Relaxed); +} + +/// Test-only read: the counter is process-wide, so tests assert on deltas. +#[cfg(test)] +pub(crate) fn sync_rejected_count() -> u64 { + SYNC_REJECTED.load(Ordering::Relaxed) +} + +/// Index of the longest prefix in `prefixes` that `key` starts with. +/// +/// Longest rather than first so that adding a narrower prefix later (say +/// `node/status/` next to `node/`) routes keys to the narrower series instead +/// of depending on array order. The current set does not overlap, so this is +/// here to keep the next addition from being a silent mis-bucketing. +fn longest_prefix_index(prefixes: &[&str], key: &str) -> Option<usize> { + prefixes + .iter() + .enumerate() + .filter(|(_, prefix)| key.starts_with(*prefix)) + .max_by_key(|(_, prefix)| prefix.len()) + .map(|(index, _)| index) +} + +fn prefix_index(key: &str) -> usize { + longest_prefix_index(&METERED_PREFIXES, key).unwrap_or(METERED_PREFIXES.len()) +} + +fn prefix_label(index: usize) -> &'static str { + METERED_PREFIXES.get(index).copied().unwrap_or(OTHER_PREFIX) +} + +/// Live state sampled for one scrape. +pub(crate) struct Snapshot { + pub version: String, + pub node_id: u32, + pub instances: u64, + pub connections: u64, + pub nodes_total: u64, + pub nodes_active: u64, + pub accel: ProxyAccelStatus, + pub stores: Vec<StoreSnapshot>, + /// domain -> certificate `notAfter`, in seconds since the epoch. + pub cert_not_after: Vec<(String, u64)>, + /// node id -> the latest time any gateway observed the node, in seconds + /// since the epoch (max across observers, future-dated reports dropped). + pub node_last_seen: Vec<(u32, u64)>, +} + +/// One WaveKV store (`persistent` or `ephemeral`). +pub(crate) struct StoreSnapshot { + pub name: &'static str, + pub keys: u64, + pub next_seq: u64, + pub dirty: bool, + pub peers: Vec<PeerSnapshot>, +} + +pub(crate) struct PeerSnapshot { + pub id: u32, + /// Highest sequence from this peer that the local store covers. + pub local_ack: u64, + /// Highest local sequence that the peer reports covering. + pub peer_ack: u64, +} + +/// Render the Prometheus text exposition format. +pub(crate) fn render(snapshot: &Snapshot) -> String { + let mut out = String::with_capacity(2048); + + gauge( + &mut out, + "dstack_gateway_build_info", + "Gateway build information.", + &format!( + "{{version=\"{}\",node_id=\"{}\"}}", + escape_label(&snapshot.version), + snapshot.node_id + ), + 1, + ); + gauge( + &mut out, + "dstack_gateway_cluster_instances", + "CVM instances currently in the routing table. Replicated: every node reports the same value, so aggregate with max(), not sum().", + "", + snapshot.instances, + ); + gauge( + &mut out, + "dstack_gateway_connections", + "Proxy connections currently open.", + "", + snapshot.connections, + ); + gauge( + &mut out, + "dstack_gateway_cluster_nodes", + "Gateway nodes known to the cluster. Replicated: aggregate with max(), not sum().", + "", + snapshot.nodes_total, + ); + gauge( + &mut out, + "dstack_gateway_cluster_nodes_active", + "Gateway nodes this node does not consider down. Replicated state seen locally, so disagreement between nodes is itself the replication-lag signal.", + "", + snapshot.nodes_active, + ); + + counter( + &mut out, + "dstack_gateway_ktls_offloaded_total", + "Connections handed to the kernel TLS ULP.", + "", + snapshot.accel.ktls_offloaded, + ); + counter( + &mut out, + "dstack_gateway_ktls_offload_failed_total", + "Connections whose kernel TLS handover failed.", + "", + snapshot.accel.ktls_offload_failed, + ); + counter( + &mut out, + "dstack_gateway_splice_engaged_total", + "Connections that entered a zero-copy splice relay.", + "", + snapshot.accel.splice_engaged, + ); + + header( + &mut out, + "dstack_gateway_cluster_kv_keys", + "Keys held in a WaveKV store. Replicated: aggregate with max(), not sum().", + "gauge", + ); + for store in &snapshot.stores { + line( + &mut out, + "dstack_gateway_cluster_kv_keys", + &store_label(store), + store.keys, + ); + } + header( + &mut out, + "dstack_gateway_kv_next_seq", + "Next sequence number this node will assign in a WaveKV store.", + "gauge", + ); + for store in &snapshot.stores { + line( + &mut out, + "dstack_gateway_kv_next_seq", + &store_label(store), + store.next_seq, + ); + } + header( + &mut out, + "dstack_gateway_kv_dirty", + "1 when a WaveKV store holds changes that are not in its snapshot.", + "gauge", + ); + for store in &snapshot.stores { + line( + &mut out, + "dstack_gateway_kv_dirty", + &store_label(store), + u64::from(store.dirty), + ); + } + + header( + &mut out, + "dstack_gateway_kv_peer_local_ack", + "How far this node has consumed a peer's log.", + "gauge", + ); + for (store, peer) in peers(snapshot) { + line( + &mut out, + "dstack_gateway_kv_peer_local_ack", + &peer_label(store, peer), + peer.local_ack, + ); + } + header( + &mut out, + "dstack_gateway_kv_peer_remote_ack", + "How far a peer reports having consumed this node's log.", + "gauge", + ); + for (store, peer) in peers(snapshot) { + line( + &mut out, + "dstack_gateway_kv_peer_remote_ack", + &peer_label(store, peer), + peer.peer_ack, + ); + } + header( + &mut out, + "dstack_gateway_kv_decode_failures_total", + "Replicated values that could not be decoded, by key prefix.", + "counter", + ); + for (index, counter) in DECODE_FAILURES.iter().enumerate() { + line( + &mut out, + "dstack_gateway_kv_decode_failures_total", + &format!("{{prefix=\"{}\"}}", escape_label(prefix_label(index))), + counter.load(Ordering::Relaxed), + ); + } + + counter( + &mut out, + "dstack_gateway_wg_reconfigure_total", + "WireGuard config applications attempted.", + "", + WG_RECONFIGURE_TOTAL.load(Ordering::Relaxed), + ); + counter( + &mut out, + "dstack_gateway_wg_reconfigure_failures_total", + "WireGuard config applications that did not reach the data plane: render failure, write failure, or a config wg syncconf rejected.", + "", + WG_RECONFIGURE_FAILURES.load(Ordering::Relaxed), + ); + counter( + &mut out, + "dstack_gateway_kv_persist_failures_total", + "Periodic WaveKV snapshots that failed.", + "", + KV_PERSIST_FAILURES.load(Ordering::Relaxed), + ); + counter( + &mut out, + "dstack_gateway_kv_wal_sync_failures_total", + "Attempts to force the WaveKV write-ahead log to disk that failed. While this is rising, writes accepted in the last wal_sync_window are not durable.", + "", + KV_WAL_SYNC_FAILURES.load(Ordering::Relaxed), + ); + counter( + &mut out, + "dstack_gateway_sync_rejected_total", + "Sync envelopes this node sent that a peer rejected with HTTP 403. This can indicate a removal lockout or an app-identity mismatch.", + "", + SYNC_REJECTED.load(Ordering::Relaxed), + ); + + header( + &mut out, + "dstack_gateway_node_last_seen_timestamp_seconds", + "Latest time any gateway observed the node, unix seconds. Replicated (max across observers): every node reports roughly the same value, so aggregate with max(), and alert on time() minus this. A node an operator removed leaves the series entirely -- absence here plus dstack_gateway_sync_rejected_total on the machine itself can indicate the removed-by-mistake picture.", + "gauge", + ); + for (node_id, last_seen) in &snapshot.node_last_seen { + line( + &mut out, + "dstack_gateway_node_last_seen_timestamp_seconds", + &format!("{{node=\"{node_id}\"}}"), + *last_seen, + ); + } + + gauge( + &mut out, + "dstack_gateway_cluster_cert_domains", + "Domains holding certificate data. Exceeding the number of cert_not_after series means the series were truncated.", + "", + snapshot.cert_not_after.len() as u64, + ); + header( + &mut out, + "dstack_gateway_cluster_cert_not_after_seconds", + "Certificate expiry per domain, in seconds since the epoch. Replicated: every node reports the same series.", + "gauge", + ); + // Domains are only created through the admin API, so in practice this is a + // handful of wildcard certificates. The cap is for the case where it is + // not: the records are replicated, so a peer with write access could turn + // one label into an unbounded series count. Truncation is by domain order + // rather than by expiry so the exported set does not flap between scrapes; + // `cert_domains` above is what tells you it happened. + for (domain, not_after) in snapshot.cert_not_after.iter().take(MAX_CERT_SERIES) { + line( + &mut out, + "dstack_gateway_cluster_cert_not_after_seconds", + &format!("{{domain=\"{}\"}}", escape_label(domain)), + *not_after, + ); + } + + out +} + +fn peers(snapshot: &Snapshot) -> impl Iterator<Item = (&StoreSnapshot, &PeerSnapshot)> { + snapshot + .stores + .iter() + .flat_map(|store| store.peers.iter().map(move |peer| (store, peer))) +} + +fn store_label(store: &StoreSnapshot) -> String { + format!("{{store=\"{}\"}}", escape_label(store.name)) +} + +fn peer_label(store: &StoreSnapshot, peer: &PeerSnapshot) -> String { + format!( + "{{store=\"{}\",peer=\"{}\"}}", + escape_label(store.name), + peer.id + ) +} + +fn header(out: &mut String, name: &str, help: &str, kind: &str) { + let _ = writeln!(out, "# HELP {name} {help}"); + let _ = writeln!(out, "# TYPE {name} {kind}"); +} + +fn line(out: &mut String, name: &str, labels: &str, value: u64) { + let _ = writeln!(out, "{name}{labels} {value}"); +} + +fn gauge(out: &mut String, name: &str, help: &str, labels: &str, value: u64) { + header(out, name, help, "gauge"); + line(out, name, labels, value); +} + +fn counter(out: &mut String, name: &str, help: &str, labels: &str, value: u64) { + header(out, name, help, "counter"); + line(out, name, labels, value); +} + +/// Escape a label value per the exposition format. +/// +/// Domains reach this from replicated state, so an unescaped quote or newline +/// would be a peer-controlled way to forge series in the scrape output. +/// +/// The format defines exactly three escapes: `\\`, `\"` and `\n`. Escaping +/// anything else is not the safer choice it looks like -- `prometheus/common`'s +/// parser, which backs `promtool check metrics` and most client tooling, +/// rejects an unknown escape sequence outright. Emitting `\t` would hand the +/// same hostile peer a cheaper attack than the one this function exists to +/// stop: one tab in a domain and the entire scrape stops parsing. Remaining +/// control characters are dropped instead, so the output is valid and carries +/// no raw control bytes either. +fn escape_label(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for ch in value.chars() { + match ch { + '\\' => escaped.push_str("\\\\"), + '"' => escaped.push_str("\\\""), + '\n' => escaped.push_str("\\n"), + _ if ch.is_control() => {} + _ => escaped.push(ch), + } + } + escaped +} + +#[cfg(test)] +mod tests { + use super::*; + + fn snapshot() -> Snapshot { + Snapshot { + version: "0.0.0-test".to_string(), + node_id: 7, + instances: 3, + connections: 12, + nodes_total: 3, + nodes_active: 2, + accel: ProxyAccelStatus { + ktls_mode: "off".to_string(), + splice_mode: "off".to_string(), + ktls_offloaded: 5, + ktls_offload_failed: 1, + splice_engaged: 4, + }, + stores: vec![StoreSnapshot { + name: "persistent", + keys: 42, + next_seq: 100, + dirty: true, + peers: vec![PeerSnapshot { + id: 2, + local_ack: 9, + peer_ack: 8, + }], + }], + cert_not_after: vec![("app.example.com".to_string(), 1_800_000_000)], + node_last_seen: vec![(2, 1_777_000_000)], + } + } + + #[test] + fn every_series_is_declared_before_it_is_used() { + let rendered = render(&snapshot()); + let mut declared = std::collections::HashSet::new(); + for row in rendered.lines() { + if let Some(rest) = row.strip_prefix("# TYPE ") { + let name = rest.split(' ').next().unwrap_or_default(); + declared.insert(name.to_string()); + continue; + } + if row.starts_with('#') { + continue; + } + let name = row + .split(['{', ' ']) + .next() + .expect("a sample line names a series"); + assert!( + declared.contains(name), + "sample {name} appears without a # TYPE line" + ); + } + } + + #[test] + fn samples_carry_the_values_they_were_given() { + let rendered = render(&snapshot()); + for expected in [ + "dstack_gateway_build_info{version=\"0.0.0-test\",node_id=\"7\"} 1", + "dstack_gateway_cluster_instances 3", + "dstack_gateway_connections 12", + "dstack_gateway_cluster_nodes_active 2", + "dstack_gateway_ktls_offload_failed_total 1", + "dstack_gateway_cluster_kv_keys{store=\"persistent\"} 42", + "dstack_gateway_kv_dirty{store=\"persistent\"} 1", + "dstack_gateway_cluster_cert_not_after_seconds{domain=\"app.example.com\"} 1800000000", + "dstack_gateway_node_last_seen_timestamp_seconds{node=\"2\"} 1777000000", + ] { + assert!(rendered.contains(expected), "missing sample: {expected}"); + } + } + + #[test] + fn only_the_three_escapes_the_format_defines_are_emitted() { + // The exposition format defines \\, \" and \n and nothing else, and + // `prometheus/common`'s parser errors on any other escape sequence. A + // tab that reaches a label value must therefore be dropped rather than + // written as `\t`, which would cost the whole scrape -- a cheaper + // attack than the injection this escaping exists to stop. + assert_eq!(escape_label("a\tb\rc\u{7}d"), "abcd"); + assert_eq!(escape_label("a\\b\"c\nd"), "a\\\\b\\\"c\\nd"); + + let mut snapshot = snapshot(); + snapshot.cert_not_after = vec![("tab\there\rand\u{7}bell".to_string(), 1_800_000_000)]; + let rendered = render(&snapshot); + for undefined in ["\\t", "\\r"] { + assert!( + !rendered.contains(undefined), + "emitted `{undefined}`, an escape the exposition format does not define" + ); + } + assert!(rendered.contains( + "dstack_gateway_cluster_cert_not_after_seconds{domain=\"tabhereandbell\"} 1800000000" + )); + } + + #[test] + fn a_hostile_domain_cannot_forge_a_series() { + let mut snapshot = snapshot(); + // A domain arrives from replicated state, so treat it as peer-supplied. + snapshot.cert_not_after = vec![( + "evil\" 1\ndstack_gateway_cluster_instances 999\n#".to_string(), + 1_800_000_000, + )]; + let rendered = render(&snapshot); + + // The injection stays inside one label value on one line: no forged + // sample line, and the real gauge still reads what it was given. + let forged = rendered + .lines() + .filter(|row| row.starts_with("dstack_gateway_cluster_instances ")) + .count(); + assert_eq!( + forged, 1, + "the payload escaped its label and became a sample" + ); + assert!(rendered + .lines() + .any(|row| row == "dstack_gateway_cluster_instances 3")); + assert!(rendered.contains( + "dstack_gateway_cluster_cert_not_after_seconds{domain=\"evil\\\" 1\\ndstack_gateway_cluster_instances 999\\n#\"} 1800000000" + )); + } + + #[test] + fn decode_failures_are_bucketed_by_key_prefix() { + assert_eq!(prefix_label(prefix_index("inst/abc")), "inst/"); + // No narrower `node/` prefix is metered, so this folds into `node/`. + assert_eq!(prefix_label(prefix_index("node/status/3")), "node/"); + assert_eq!(prefix_label(prefix_index("cert/example.com/data")), "cert/"); + assert_eq!(prefix_label(prefix_index("__peer_addr/3")), "__peer_addr/"); + assert_eq!( + prefix_label(prefix_index("global/certbot_config")), + "global/" + ); + // A key a peer invented does not get a series of its own. + assert_eq!(prefix_label(prefix_index("whatever/1")), "other"); + assert_eq!(prefix_label(prefix_index("")), "other"); + } + + #[test] + fn a_narrower_prefix_wins_over_a_wider_one() { + // The metered set does not overlap today, so drive the rule directly: + // adding `node/status/` later must not depend on where in the array it + // lands. + let prefixes = ["node/", "node/status/"]; + assert_eq!(longest_prefix_index(&prefixes, "node/status/3"), Some(1)); + assert_eq!(longest_prefix_index(&prefixes, "node/info/3"), Some(0)); + + let reversed = ["node/status/", "node/"]; + assert_eq!(longest_prefix_index(&reversed, "node/status/3"), Some(0)); + + assert_eq!(longest_prefix_index(&prefixes, "inst/1"), None); + } + + #[test] + fn the_per_domain_expiry_series_is_capped() { + let mut snapshot = snapshot(); + let total = MAX_CERT_SERIES + 25; + snapshot.cert_not_after = (0..total) + .map(|i| (format!("d{i:04}.example.com"), 1_800_000_000 + i as u64)) + .collect(); + let rendered = render(&snapshot); + + let exported = rendered + .lines() + .filter(|row| row.starts_with("dstack_gateway_cluster_cert_not_after_seconds{")) + .count(); + assert_eq!(exported, MAX_CERT_SERIES, "the cap did not hold"); + // The real count still reaches the operator, so truncation is visible + // rather than silent. + assert!(rendered + .lines() + .any(|row| row == format!("dstack_gateway_cluster_cert_domains {total}"))); + } + + #[test] + fn a_scrape_does_not_feed_the_counter_it_reports() { + // Process-wide statics: assert on deltas, never on absolute values. + let bucket = &DECODE_FAILURES[prefix_index("conn/probe")]; + let before = bucket.load(Ordering::Relaxed); + { + let _guard = scrape_guard(); + record_decode_failure("conn/probe"); + // Nested guards must not end suppression early. + { + let _inner = scrape_guard(); + record_decode_failure("conn/probe"); + } + record_decode_failure("conn/probe"); + } + assert_eq!( + bucket.load(Ordering::Relaxed), + before, + "a scrape counted its own reads" + ); + + record_decode_failure("conn/probe"); + assert_eq!( + bucket.load(Ordering::Relaxed), + before + 1, + "suppression outlived the scrape" + ); + } + + #[test] + fn recording_a_failure_moves_its_own_bucket_only() { + // Process-wide statics: assert on deltas, never on absolute values. + let before: Vec<u64> = DECODE_FAILURES + .iter() + .map(|counter| counter.load(Ordering::Relaxed)) + .collect(); + record_decode_failure("dns_cred/abc"); + let after: Vec<u64> = DECODE_FAILURES + .iter() + .map(|counter| counter.load(Ordering::Relaxed)) + .collect(); + + let moved = prefix_index("dns_cred/abc"); + for (index, (before, after)) in before.iter().zip(after.iter()).enumerate() { + let expected = if index == moved { before + 1 } else { *before }; + assert_eq!(*after, expected, "bucket {} moved unexpectedly", index); + } + } +} diff --git a/dstack/gateway/src/models.rs b/dstack/gateway/src/models.rs index 77d2cfe2e..63564c4e1 100644 --- a/dstack/gateway/src/models.rs +++ b/dstack/gateway/src/models.rs @@ -2,11 +2,9 @@ // // SPDX-License-Identifier: Apache-2.0 -use dstack_gateway_rpc::{AcmeInfoResponse, StatusResponse}; +use dstack_gateway_rpc::{AcmeInfoResponse, ProxyAccelStatus, StatusResponse}; use rinja::Template; -use serde::{Deserialize, Serialize}; use std::{ - collections::{btree_map::Iter, BTreeMap}, net::Ipv4Addr, sync::{ atomic::{AtomicU64, Ordering}, @@ -23,39 +21,19 @@ mod filters { } } -pub struct MapValues<'a, K, V>(pub &'a BTreeMap<K, V>); -impl<K, V> Copy for MapValues<'_, K, V> {} -impl<K, V> Clone for MapValues<'_, K, V> { - fn clone(&self) -> Self { - *self - } -} -impl<'a, K, V> From<&'a BTreeMap<K, V>> for MapValues<'a, K, V> { - fn from(map: &'a BTreeMap<K, V>) -> Self { - MapValues(map) - } -} - -pub struct MapValuesIter<'a, K, V>(Iter<'a, K, V>); - -impl<'a, K, V> IntoIterator for MapValues<'a, K, V> { - type Item = &'a V; - type IntoIter = MapValuesIter<'a, K, V>; - - fn into_iter(self) -> Self::IntoIter { - MapValuesIter(self.0.iter()) - } -} - -impl<'a, K, V> Iterator for MapValuesIter<'a, K, V> { - type Item = &'a V; - - fn next(&mut self) -> Option<Self::Item> { - self.0.next().map(|(_, v)| v) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] +/// One instance as the data plane sees it: what the CVM reported in its `inst/` +/// record, what an operator decided under `admin/`, and this node's live +/// connection count. +/// +/// Not a stored shape, which is the difference from +/// [`crate::kv::InstanceRecord`] -- that one is the `inst/` record and nothing +/// else. Assembling three sources into one type here is what lets the proxy +/// answer a routing question without touching the store; keeping them apart is +/// what stops a registration, which writes the `inst/` record in full, from +/// carrying an operator's decision back out with it. +/// `From<&InstanceInfo> for InstanceRecord` is that narrowing, and the compiler +/// is what checks it drops exactly the right fields. +#[derive(Clone, Debug)] pub struct InstanceInfo { pub id: String, pub app_id: String, @@ -64,24 +42,348 @@ pub struct InstanceInfo { pub reg_time: SystemTime, /// Port policy. `None` means the CVM didn't report any (legacy); /// gateway will lazily populate via Info() on first proxied connection. - #[serde(default)] pub port_policy: Option<PortPolicy>, /// Hex-encoded compose_hash that `port_policy` was learned against. The /// cache is invalidated when a new registration presents a different hash. - #[serde(default)] pub port_policy_hash: String, /// Operator-set override (Admin RPC). Takes precedence over `port_policy` /// when set; survives app upgrades. - #[serde(default)] + /// + /// Persisted under `admin/<instance_id>/port_policy`, not in the instance + /// record -- see [`crate::kv::PortPolicyOverride`]. pub admin_port_policy: Option<PortPolicy>, - #[serde(skip)] + /// Operator-set traffic gate (Admin RPC). `None` means no operator ever + /// touched it. See [`InstanceInfo::is_ready`]. + /// + /// Persisted under `admin/<instance_id>/ready`, not in the instance record + /// -- see [`crate::kv::KvStore::instance_gate`]. + pub ready: Option<bool>, + /// What this CVM asked for, and what this node has observed since. + /// + /// Only the declaration crosses the store, as the bare `health_check` + /// boolean of the `inst/` record: it came from the CVM, so it belongs + /// there with everything else the CVM said about itself. The observation + /// does not, deliberately -- every node polls for itself, the same way + /// each node reads its own WireGuard handshakes. A shared "healthy" flag + /// would outlive the instance that set it, and "I could not reach it" is a + /// per-node fact anyway. + /// + /// The field is crate-visible so a record can be built in one expression; + /// [`Health`]'s own state is not, so there is no way to build one whose + /// declaration and verdict disagree. + pub(crate) health: Health, pub connections: Arc<AtomicU64>, } +/// What a CVM reported about itself when it registered. +/// +/// Bundled rather than passed as positional arguments: every one of these is a +/// self-declared capability, and the list has already grown once. A struct +/// keeps the next addition from rippling through every call site. +#[derive(Debug, Clone, Default)] +pub struct ReportedCapabilities { + /// Per-port behaviour declared by the app. `None` means "not reported" + /// (legacy CVM), and the gateway fetches it lazily instead. + pub port_policy: Option<PortPolicy>, + /// Whether the app asked for its traffic to be gated on its health + /// (`requirements.health_check`). + /// + /// `None` means the caller has nothing to say about it -- the debug + /// registration path, which has no CVM to ask -- and leaves whatever the + /// instance already declared in place. `Some(false)` is an app that did not + /// opt in, or an image that predates the field; the gateway polls neither. + pub health_check: Option<bool>, +} + +impl From<Option<PortPolicy>> for ReportedCapabilities { + /// Convenience for callers that only have a port policy to report and + /// nothing to say about health gating. + fn from(port_policy: Option<PortPolicy>) -> Self { + Self { + port_policy, + health_check: None, + } + } +} + +/// What this gateway node knows about an instance's application-level health. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum HealthState { + /// No poll has completed yet. Reads as *not* healthy: a CVM registers + /// during boot, before its containers exist, so "not asked yet" is far + /// more often "not up yet" than "fine". + #[default] + Unknown, + /// The guest agent reported that every container declaring a healthcheck + /// is healthy. + Healthy, + /// The guest agent reported at least one container not healthy, or could + /// not be reached at all. + Unhealthy, + /// This instance never asked to be gated -- an app that did not set + /// `requirements.health_check`, or an image that predates the + /// field. Reads as healthy: it has to keep serving exactly as before. + Ungated, +} + +/// An instance's health gating: what the CVM asked for, and what this node has +/// observed since. +/// +/// One value rather than two fields, because the second is a function of the +/// first whenever there is no observation to report -- `Ungated` is not a +/// verdict, it is a restatement of "never asked". Kept apart, that derivation +/// has to be repeated at every site that builds or resets a record, and a site +/// that forgets produces an instance which is simultaneously routable and +/// unpollable, with nothing on this node able to lift it. +/// +/// Neither half is serialized. The declaration reaches the store as the bare +/// boolean of [`crate::kv::InstanceRecord::health_check`], through +/// `From<&InstanceInfo>`; the observation reaches it not at all. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Health { + gated: bool, + state: HealthState, +} + +impl Default for Health { + /// Not derived. `HealthState::default()` is `Unknown`, but the default + /// *declaration* is "not gated", whose state is `Ungated` -- and a derive + /// would pair the two independently and produce an instance that is held + /// out of rotation while also being filtered out of the poll set, with + /// nothing able to lift it. Deferring to `From<bool>` keeps one derivation. + fn default() -> Self { + Self::from(false) + } +} + +impl From<bool> for Health { + fn from(gated: bool) -> Self { + Self { + gated, + state: HealthState::initial(gated), + } + } +} + +impl Health { + /// Whether this instance asked to be gated, and therefore polled. + pub fn is_gated(self) -> bool { + self.gated + } + + /// The last observation, or the state that stands in for never having one. + pub fn state(self) -> HealthState { + self.state + } + + /// Record what a poll found. + fn observe(&mut self, state: HealthState) { + self.state = state; + } + + /// Apply what the CVM declared at registration. + /// + /// A declaration that changed means the image did, so any verdict about the + /// previous one is void. An unchanged one must not disturb a live verdict: + /// a CVM re-registers every three minutes without going anywhere, and + /// resetting on each would drop a healthy instance out of rotation until + /// its next poll, forever, on a timer. + fn declare(&mut self, gated: bool) { + if self.gated != gated { + self.gated = gated; + self.forget(); + } + } + + /// Drop any verdict, as if the instance had just registered. + fn forget(&mut self) { + self.state = HealthState::initial(self.gated); + } +} + +impl HealthState { + /// The state an instance starts in, given whether it asked to be gated. + /// + /// An instance that opted in starts `Unknown` -- held out of rotation until + /// a poll answers -- because registration happens during boot, before the + /// app exists. One that did not starts `Ungated` and stays eligible, so + /// apps that never asked for this keep working. + fn initial(health_check: bool) -> Self { + if health_check { + HealthState::Unknown + } else { + HealthState::Ungated + } + } + + /// Whether this state permits traffic. + pub fn is_healthy(self) -> bool { + matches!(self, HealthState::Healthy | HealthState::Ungated) + } + + /// Stable lowercase name, as reported over the admin API. + pub fn as_str(self) -> &'static str { + match self { + HealthState::Unknown => "unknown", + HealthState::Healthy => "healthy", + HealthState::Unhealthy => "unhealthy", + HealthState::Ungated => "ungated", + } + } +} + impl InstanceInfo { pub fn num_connections(&self) -> u64 { self.connections.load(Ordering::Relaxed) } + + /// Whether an operator has left this instance eligible for traffic. + /// + /// Defaults to ready, so the gate is invisible until someone explicitly + /// closes it -- an instance that predates this field, or one nobody has + /// touched, keeps serving exactly as before. + /// + /// This only gates *multi-target* selection, i.e. connections addressed to + /// the app id. Routing to the instance id directly ignores it on purpose: + /// the point of taking an instance out of rotation is to investigate it + /// while it is still running, which needs the instance to stay reachable. + pub fn is_ready(&self) -> bool { + self.ready.unwrap_or(true) + } + + /// Whether this node's last health observation permits traffic. + /// + /// Unlike [`InstanceInfo::is_ready`], this is an inference and can be + /// wrong, so callers fail open when it would empty an app's candidate set. + pub fn is_healthy(&self) -> bool { + self.health.state().is_healthy() + } + + /// This node's last health observation, for reporting. + pub fn health(&self) -> HealthState { + self.health.state() + } + + /// Whether this instance asked to be gated, and polled, at all. + pub fn health_check(&self) -> bool { + self.health.is_gated() + } + + /// Record a fresh observation. + pub fn set_health(&mut self, state: HealthState) { + self.health.observe(state); + } + + /// Apply what the CVM declared, re-deriving the verdict if it changed. + pub fn set_health_check(&mut self, health_check: bool) { + self.health.declare(health_check); + } + + /// Forget any verdict, as if the instance had just registered. + /// + /// For the caller that can see a *reboot* -- a new WireGuard key -- which + /// the declaration alone cannot show. + pub fn reset_health(&mut self) { + self.health.forget(); + } + + /// Adopt another record's verdict, when it is about the same running app. + /// + /// Used by the reload path: health is this node's own observation and the + /// store does not carry it, so re-deriving it from the record would reset + /// the whole fleet to `Unknown` on every sync round. Two things must not + /// survive. A changed declaration means the image was replaced. And a + /// changed WireGuard key means the CVM rebooted: the guest caches its key + /// store in `/run`, which is tmpfs, so a fresh key is a fresh boot. Without + /// the second check the reboot reset is only applied on whichever node took + /// the registration -- every *other* node learns about the new boot through + /// sync and quietly copies its own pre-reboot verdict onto it. + pub fn inherit_health_from(&mut self, previous: &Self) { + if self.health_check() == previous.health_check() && self.public_key == previous.public_key + { + self.health.observe(previous.health.state()); + } + } + + /// Whether replacing `self` with `other` could invalidate a cached + /// selection. + /// + /// `ip` is copied straight into the cached `AddressInfo`; `app_id` decides + /// which cache entry the instance belongs to; `public_key` is the key the + /// handshake freshness filter looks up; `ready` and `health` both + /// gate eligibility. A change to any of them means a cached `top_n` was + /// computed against a record that no longer exists. + /// + /// `health` is this node's own observation rather than something the record + /// carries, and a reload copies it across, so it only differs here when the + /// reload had to reset it -- the declared capability changed, meaning the + /// image did. + pub fn routing_inputs_differ(&self, other: &Self) -> bool { + self.routing_inputs() != other.routing_inputs() + } + + /// The parts of this record a cached selection was computed from. + /// + /// Borrows rather than clones, so an in-place edit can be checked for + /// staleness without copying a record it may not have changed -- this runs + /// once per poll result, which is the whole fleet every interval. + pub(crate) fn routing_inputs(&self) -> RoutingInputs<'_> { + RoutingInputs { + ip: self.ip, + app_id: &self.app_id, + public_key: &self.public_key, + ready: self.ready, + health: self.health, + } + } +} + +/// A borrowed view of the fields [`InstanceInfo::routing_inputs_differ`] +/// compares, so that definition lives in exactly one place. Adding a field to +/// the selection means adding it here; nothing else has to be audited. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct RoutingInputs<'a> { + ip: Ipv4Addr, + app_id: &'a str, + public_key: &'a str, + ready: Option<bool>, + health: Health, +} + +impl RoutingInputs<'_> { + /// Detach from the record, so a caller can compare across a mutation. + pub(crate) fn to_owned(self) -> OwnedRoutingInputs { + OwnedRoutingInputs { + ip: self.ip, + app_id: self.app_id.to_string(), + public_key: self.public_key.to_string(), + ready: self.ready, + health: self.health, + } + } +} + +/// [`RoutingInputs`] with the borrows taken, for comparing a record against +/// itself either side of an edit. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct OwnedRoutingInputs { + ip: Ipv4Addr, + /// Which cache entry the record belonged to *before* the edit. A record + /// that changed apps invalidates both. + pub(crate) app_id: String, + public_key: String, + ready: Option<bool>, + health: Health, +} + +impl PartialEq<RoutingInputs<'_>> for OwnedRoutingInputs { + fn eq(&self, other: &RoutingInputs<'_>) -> bool { + self.ip == other.ip + && self.app_id == other.app_id + && self.public_key == other.public_key + && self.ready == other.ready + && self.health == other.health + } } /// Snapshot of an instance's port-policy state for admin inspection. @@ -155,12 +457,22 @@ impl<C: Counting> Drop for EnteredCounter<C> { } } +/// One `[Peer]` stanza of the rendered WireGuard config. +/// +/// Built by the caller rather than borrowed straight from the instance table so +/// that a record `wg` would refuse can be left out — the template renders with +/// `escape = "none"`, and `wg syncconf` rejects the whole file on one bad line. +pub struct WgPeer<'a> { + pub public_key: &'a str, + pub ip: Ipv4Addr, +} + #[derive(Template)] #[template(path = "wg.conf", escape = "none")] pub struct WgConf<'a> { pub private_key: &'a str, pub listen_port: u16, - pub peers: MapValues<'a, String, InstanceInfo>, + pub peers: Vec<WgPeer<'a>>, } #[derive(Template)] @@ -168,4 +480,54 @@ pub struct WgConf<'a> { pub struct Dashboard { pub status: StatusResponse, pub acme_info: AcmeInfoResponse, + /// Lifted out of `status` so the template does not have to unwrap the + /// proto's optional message on every field. + pub accel: ProxyAccelStatus, +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Only the declaration survives a trip through the store, which is what + /// `InstanceRecord::health_check` carries and what a reload rebuilds from. + #[test] + fn only_the_declaration_survives_a_round_trip() { + let restored = Health::from(true); + assert!(restored.is_gated()); + // Not `Healthy`. The observation is deliberately not carried, so a + // record that arrives from the store has to start held out of rotation + // rather than inherit a verdict nobody made. + assert_eq!(restored.state(), HealthState::Unknown); + } + + /// The pair cannot be built inconsistently, which is the point of it being + /// one value. `HealthState::default()` is `Unknown` while the default + /// declaration is "not gated", whose state is `Ungated` -- a derived + /// `Default` would pair those and produce an instance that is both held out + /// of rotation and filtered out of the poll set, with nothing able to lift + /// it. + #[test] + fn the_default_declaration_is_not_the_default_state() { + assert_eq!(Health::default(), Health::from(false)); + assert_eq!(Health::default().state(), HealthState::Ungated); + assert!(Health::default().state().is_healthy()); + } + + /// A CVM re-registers every three minutes without having gone anywhere. + /// Re-deriving the verdict on each would drop a healthy instance out of + /// rotation until its next poll, forever, on a timer. + #[test] + fn redeclaring_the_same_intent_leaves_a_verdict_alone() { + let mut health = Health::from(true); + health.observe(HealthState::Healthy); + + health.declare(true); + assert_eq!(health.state(), HealthState::Healthy); + + // A changed declaration means a different image, so the verdict is + // about something that is no longer running. + health.declare(false); + assert_eq!(health.state(), HealthState::Ungated); + } } diff --git a/dstack/gateway/src/pp.rs b/dstack/gateway/src/pp.rs index f6c6e09f2..893e3f844 100644 --- a/dstack/gateway/src/pp.rs +++ b/dstack/gateway/src/pp.rs @@ -251,7 +251,7 @@ mod tests { // PROXY prefix matched but no \r\n terminator within V1_MAX_LENGTH bytes. let bytes = vec![b'P'; V1_MAX_LENGTH + 8]; // all 'P' — never closes let mut head = b"PROXY".to_vec(); - head.extend(std::iter::repeat(b'A').take(V1_MAX_LENGTH)); + head.extend(std::iter::repeat_n(b'A', V1_MAX_LENGTH)); let err = read_proxy_header(&head[..]).await.unwrap_err(); let msg = format!("{err:#}"); assert!( diff --git a/dstack/gateway/src/proxy.rs b/dstack/gateway/src/proxy.rs index 4451ee359..25808b687 100644 --- a/dstack/gateway/src/proxy.rs +++ b/dstack/gateway/src/proxy.rs @@ -12,6 +12,7 @@ use std::{ }; use anyhow::{bail, Context, Result}; +use http_client::ConnectionReuse; use or_panic::ResultOrPanic; use sni::extract_sni; pub(crate) use tls_passthough::AppAddressResolver; @@ -22,7 +23,7 @@ use tokio::{ runtime::Runtime, time::timeout, }; -use tracing::{debug, error, info, info_span, Instrument}; +use tracing::{debug, debug_span, error, info, warn, Instrument}; use crate::{ config::ProxyConfig, @@ -42,9 +43,48 @@ pub(crate) struct AddressInfo { pub(crate) type AddressGroup = smallvec::SmallVec<[AddressInfo; 4]>; +mod adaptive_ktls; +mod balance; +/// Largest response the gateway will read from a guest agent. +/// +/// `Worker.Health` returns a bool and a short list, so this is generous for the +/// poller. It is sized for the other guest-facing call: `DstackGuest.Info` +/// carries `tcb_info`, which embeds the app's entire `app_compose` -- including +/// `docker_compose_file` verbatim. Nothing caps how large an app-compose may +/// be, and a legacy CVM whose `Info` exceeds this never gets a port policy and +/// is denied permanently, so the bound has to sit well above any plausible +/// compose file rather than merely above a health response. +const MAX_GUEST_RESPONSE_BYTES: usize = 16 * 1024 * 1024; + +/// A prpc client for a CVM's guest agent. +/// +/// The one place that decides how the gateway talks to a guest agent, because +/// that peer is the one dstack's threat model does not trust -- the reason +/// health is polled rather than pushed. Bounding the response is a per-caller +/// opt-in in `http-client` (a transport-wide cap would break `dstack vmm +/// logs`), so it has to be applied somewhere; applying it here means a new +/// guest-facing client cannot forget. +/// +/// `reuse` is an argument rather than a default for the opposite reason: the +/// two guest-facing callers want different answers and neither is obviously +/// right, so the compiler asks. See [`ConnectionReuse`]. +pub(crate) fn guest_agent_client( + url: String, + reuse: ConnectionReuse, +) -> http_client::prpc::PrpcClient { + http_client::prpc::PrpcClient::new(url) + .with_max_response_bytes(MAX_GUEST_RESPONSE_BYTES) + .with_connection_reuse(reuse) +} + +pub(crate) mod health_check; +mod idle; mod io_bridge; pub(crate) mod port_policy; +mod reuseport; mod sni; +mod splice; +pub(crate) mod stats; mod tls_passthough; mod tls_terminate; @@ -141,7 +181,7 @@ async fn handle_connection(inbound: TcpStream, state: Proxy) -> Result<()> { .await .context("proxy protocol header timeout")? .context("failed to read proxy protocol header")?; - info!("client address: {}", DisplayAddr(&pp_header)); + debug!("client address: {}", DisplayAddr(&pp_header)); let (sni, buffer) = timeout(timeouts.handshake, take_sni(&mut inbound)) .await @@ -168,64 +208,172 @@ async fn handle_connection(inbound: TcpStream, state: Proxy) -> Result<()> { } } -#[inline(never)] -pub async fn proxy_main(rt: &Runtime, config: &ProxyConfig, proxy: Proxy) -> Result<()> { +/// Bind one listener per configured port. +/// +/// With `reuse_port` every worker binds its own listener on the same port and +/// the kernel spreads incoming connections across them, so each worker can +/// accept and serve its connections without any cross-thread handoff. +/// Accept queue depth. tokio's default is 1024, which is where SYN drops start +/// under connection bursts; both listen paths use this so they behave alike. +const LISTEN_BACKLOG: i32 = 4096; + +async fn bind_listeners(config: &ProxyConfig, reuse_port: bool) -> Result<Vec<TcpListener>> { let mut tcp_listeners = Vec::new(); for &port in &config.listen_port { - let listener = TcpListener::bind((config.listen_addr, port)) - .await - .with_context(|| format!("failed to bind {}:{}", config.listen_addr, port))?; + let listener = { + let addr = std::net::SocketAddr::from((config.listen_addr, port)); + let socket = socket2::Socket::new( + socket2::Domain::IPV4, + socket2::Type::STREAM, + Some(socket2::Protocol::TCP), + ) + .context("failed to create listening socket")?; + if reuse_port { + socket + .set_reuse_port(true) + .context("failed to set SO_REUSEPORT")?; + } + socket.set_reuse_address(true).ok(); + socket.set_nonblocking(true).ok(); + socket + .bind(&addr.into()) + .with_context(|| format!("failed to bind {addr}"))?; + socket.listen(LISTEN_BACKLOG).context("failed to listen")?; + TcpListener::from_std(std::net::TcpListener::from(socket)) + .context("failed to register listener with tokio")? + }; info!("tcp bridge listening on {}:{}", config.listen_addr, port); tcp_listeners.push(listener); } + Ok(tcp_listeners) +} + +#[inline(never)] +pub async fn proxy_main(rt: &Runtime, config: &ProxyConfig, proxy: Proxy) -> Result<()> { + let tcp_listeners = bind_listeners(config, false).await?; + accept_loop(tcp_listeners, proxy, Some(rt), None).await +} + +/// The per-connection task: everything a single proxied connection does. +/// Was this failure just the peer hanging up? +/// +/// Clients disconnecting mid-connection is routine -- a browser navigating away, +/// a mobile network dropping, a load generator ending its run. Logging those at +/// error level buries the failures that are actually the gateway's fault: a 40 +/// minute soak produced 379 such lines and no real errors. +fn is_peer_disconnect(err: &anyhow::Error) -> bool { + err.chain().any(|cause| { + cause.downcast_ref::<std::io::Error>().is_some_and(|io| { + matches!( + io.kind(), + std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::UnexpectedEof + | std::io::ErrorKind::ConnectionAborted + ) + }) + }) +} + +fn conn_task( + inbound: TcpStream, + from: std::net::SocketAddr, + proxy: Proxy, + slot: Option<balance::CoreSlot>, +) -> impl std::future::Future<Output = ()> + Send + 'static { + let span = debug_span!("conn", id = next_connection_id()); + let conn_entered = EnteredCounter::new(&NUM_CONNECTIONS); + async move { + let _conn_entered = conn_entered; + let _slot = slot; + debug!(%from, "new connection"); + let timeouts = &proxy.config.proxy.timeouts; + match timeout(timeouts.total, handle_connection(inbound, proxy)).await { + Ok(Ok(_)) => debug!("connection closed"), + Ok(Err(e)) if is_peer_disconnect(&e) => debug!("peer disconnected: {e:#}"), + Ok(Err(e)) => error!("connection error: {e:#}"), + Err(_) => error!("connection kept too long, force closing"), + } + } + .instrument(span) +} + +/// Accept connections forever. +/// +/// `rt` selects where connections run: `Some(worker_rt)` hands them to a shared +/// multi-threaded runtime, `None` keeps them on the calling thread's own +/// runtime (thread-per-core), which avoids the cross-thread handoff and the +/// work-stealing migrations that come with it. +async fn accept_loop( + tcp_listeners: Vec<TcpListener>, + proxy: Proxy, + rt: Option<&Runtime>, + mut balance: Option<( + balance::Balancer, + tokio::sync::mpsc::Receiver<balance::Handoff>, + )>, +) -> Result<()> { if tcp_listeners.is_empty() { bail!("no tcp listen ports configured"); } - let poll_counter = AtomicUsize::new(0); loop { // Accept from any TCP listener via round-robin poll. let poll_start = poll_counter.fetch_add(1, Ordering::Relaxed); let n = tcp_listeners.len(); - let accepted: std::io::Result<(TcpStream, std::net::SocketAddr)> = - std::future::poll_fn(|cx| { - for j in 0..n { - let i = (poll_start + j) % n; - if let Poll::Ready(result) = tcp_listeners[i].poll_accept(cx) { - return Poll::Ready(result); + let accept_next = std::future::poll_fn(|cx| { + for j in 0..n { + let i = (poll_start + j) % n; + if let Poll::Ready(result) = tcp_listeners[i].poll_accept(cx) { + return Poll::Ready(result); + } + } + Poll::Pending + }); + // Also take connections other cores decided to give us. + let accepted: std::io::Result<(TcpStream, std::net::SocketAddr)> = match balance.as_mut() { + Some((_, rx)) => { + tokio::select! { + r = accept_next => r, + Some((raw, from, slot)) = rx.recv() => { + // Register the handed-over socket with *this* core's reactor. + match TcpStream::from_std(raw) { + Ok(stream) => { + let task = conn_task(stream, from, proxy.clone(), Some(slot)); + tokio::spawn(task); + } + Err(e) => error!("failed to adopt handed-over connection: {e}"), + } + continue; } } - Poll::Pending - }) - .await; + } + None => accept_next.await, + }; match accepted { Ok((inbound, from)) => { - let span = info_span!("conn", id = next_connection_id()); - let _enter = span.enter(); - let conn_entered = EnteredCounter::new(&NUM_CONNECTIONS); - - info!(%from, "new connection"); - let proxy = proxy.clone(); - rt.spawn( - async move { - let _conn_entered = conn_entered; - let timeouts = &proxy.config.proxy.timeouts; - let result = - timeout(timeouts.total, handle_connection(inbound, proxy)).await; - match result { - Ok(Ok(_)) => { - info!("connection closed"); - } - Ok(Err(e)) => { - error!("connection error: {e:#}"); - } - Err(_) => { - error!("connection kept too long, force closing"); - } + // Disable Nagle: this is a latency-sensitive proxy and small + // request/response traffic otherwise stalls on delayed ACKs. + let _ = inbound.set_nodelay(true); + // In thread-per-core mode, hand the connection to a less loaded + // core when this one is running ahead; SO_REUSEPORT's hash can + // leave a core starved and it cannot be fixed later. + let placed: Option<(TcpStream, Option<balance::CoreSlot>)> = match balance.as_ref() + { + Some((b, _)) => b.place(inbound, from).map(|(s, slot)| (s, Some(slot))), + None => Some((inbound, None)), + }; + if let Some((inbound, slot)) = placed { + let task = conn_task(inbound, from, proxy.clone(), slot); + match rt { + Some(rt) => { + rt.spawn(task); + } + None => { + tokio::spawn(task); } } - .in_current_span(), - ); + } } Err(e) => { error!("failed to accept connection: {e:?}"); @@ -240,6 +388,26 @@ fn next_connection_id() -> usize { } pub fn start(config: ProxyConfig, app_state: Proxy) -> Result<()> { + if config.thread_per_core { + // Probe SO_REUSEPORT before committing: it is the one prerequisite the + // thread-per-core model cannot work without, and failing to serve at all + // is far worse than losing the optimisation. + match probe_reuse_port(&config) { + Ok(()) => return start_thread_per_core(config, app_state), + Err(err) => warn!( + "thread_per_core requested but SO_REUSEPORT is unavailable ({err:#}); \ + falling back to the shared-runtime proxy" + ), + } + } + if config.connection_rebalance { + // It is only wired up by the thread-per-core path: there is nothing to + // rebalance between when every connection lands on a shared runtime. + warn!( + "connection_rebalance is set but has no effect without thread_per_core; \ + the shared-runtime proxy balances connections through its scheduler" + ); + } std::thread::Builder::new() .name("proxy-main".to_string()) .spawn(move || { @@ -268,10 +436,216 @@ pub fn start(config: ProxyConfig, app_state: Proxy) -> Result<()> { Ok(()) } +/// Check that a `SO_REUSEPORT` listener can actually be created and bound. +fn probe_reuse_port(config: &ProxyConfig) -> Result<()> { + let port = *config + .listen_port + .first() + .context("no tcp listen ports configured")?; + let addr = std::net::SocketAddr::from((config.listen_addr, port)); + let socket = socket2::Socket::new( + socket2::Domain::IPV4, + socket2::Type::STREAM, + Some(socket2::Protocol::TCP), + ) + .context("failed to create probe socket")?; + socket + .set_reuse_port(true) + .context("SO_REUSEPORT not supported")?; + socket.set_reuse_address(true).ok(); + socket + .bind(&addr.into()) + .with_context(|| format!("failed to bind {addr} with SO_REUSEPORT"))?; + Ok(()) +} + +/// Turn kTLS off when the kernel cannot provide it. +/// +/// Without the TLS ULP an immediate offload fails every handshake, which is at +/// least loud. A gated offload is worse: the connection is served from +/// userspace up to the threshold and only then fails, so the client gets a +/// successful response truncated at exactly the gate. Both are worse than never +/// offloading, and whether the ULP is there is not something the config can +/// know -- so ask the kernel once, at startup, before the acceptor is built +/// (that is also what decides whether rustls extracts session secrets at all). +/// +/// This only establishes that the ULP exists. A cipher suite the kernel does +/// not implement still fails per connection, at offload time. +pub fn disable_ktls_if_unsupported(config: &mut ProxyConfig) { + if config.ktls.is_none() { + return; + } + if let Err(err) = probe_ktls() { + warn!( + "kTLS is configured but unavailable ({err:#}); \ + falling back to userspace TLS record encryption" + ); + config.ktls = None; + stats::mark_ktls_unsupported(); + } +} + +/// Check that the kernel exposes the TLS upper-layer protocol. +/// +/// `TCP_ULP` is only accepted on an established socket, so this sets up a +/// throwaway loopback connection instead of probing a fresh one, which would +/// fail with `ENOTCONN` whether or not the ULP exists. +#[cfg(target_os = "linux")] +fn probe_ktls() -> Result<()> { + use std::net::{Ipv4Addr, TcpListener, TcpStream}; + use std::os::fd::AsRawFd; + + /// `include/uapi/linux/tcp.h`; not exposed by the `libc` crate. + const TCP_ULP: libc::c_int = 31; + + let listener = + TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).context("failed to bind the probe listener")?; + let addr = listener + .local_addr() + .context("failed to read the probe listener address")?; + let client = TcpStream::connect(addr).context("failed to connect the probe socket")?; + let _server = listener + .accept() + .context("failed to accept the probe socket")?; + + let name = c"tls"; + // SAFETY: `name` outlives the call and `len` matches it, NUL included. + let rc = unsafe { + libc::setsockopt( + client.as_raw_fd(), + libc::IPPROTO_TCP, + TCP_ULP, + name.as_ptr().cast(), + name.to_bytes_with_nul().len() as libc::socklen_t, + ) + }; + if rc != 0 { + return Err(std::io::Error::last_os_error()) + .context("kernel rejected the TLS ULP (is CONFIG_TLS enabled?)"); + } + Ok(()) +} + +#[cfg(not(target_os = "linux"))] +fn probe_ktls() -> Result<()> { + bail!("kernel TLS is only available on Linux") +} + +/// Thread-per-core proxy: `workers` threads, each with its own single-threaded +/// runtime and its own `SO_REUSEPORT` listener. +/// +/// The kernel load-balances new connections across the listeners, and a +/// connection is accepted and served entirely on one thread. That removes the +/// accept-thread -> worker handoff and the work-stealing scheduler's task +/// migrations, which together accounted for ~0.6 context switches per request +/// in the default model. +fn start_thread_per_core(config: ProxyConfig, app_state: Proxy) -> Result<()> { + let workers = config.workers.max(1); + // Bind every listener here, in order, rather than letting each thread bind + // its own. The CBPF steering program that originally needed a known group + // order is gone (see `super::reuseport`), but binding in one place is kept: + // it keeps listener-to-core assignment deterministic across restarts, and it + // fails startup as a whole if any bind fails, instead of leaving some cores + // serving and others dead. + let mut per_worker: Vec<Vec<std::net::TcpListener>> = + (0..workers).map(|_| Vec::new()).collect(); + for &port in &config.listen_port { + let addr = std::net::SocketAddr::from((config.listen_addr, port)); + let group = reuseport::bind_group(addr, workers, LISTEN_BACKLOG) + .with_context(|| format!("failed to bind reuseport group on {addr}"))?; + info!( + "tcp bridge listening on {}:{} across {} listeners", + config.listen_addr, port, workers + ); + for (i, l) in group.into_iter().enumerate() { + per_worker[i].push(l); + } + } + + // Per-core balancers share the connection counts; each core also gets the + // receiving end of its handoff channel. + let (balancers, receivers) = if config.connection_rebalance { + let (b, r) = balance::Balancer::build(workers); + ( + b.into_iter().map(Some).collect(), + r.into_iter().map(Some).collect(), + ) + } else { + ( + (0..workers).map(|_| None).collect::<Vec<_>>(), + (0..workers).map(|_| None).collect::<Vec<_>>(), + ) + }; + + let config = Arc::new(config); + for (i, ((std_listeners, balancer), receiver)) in per_worker + .into_iter() + .zip(balancers) + .zip(receivers) + .enumerate() + { + let config = config.clone(); + let app_state = app_state.clone(); + std::thread::Builder::new() + .name(format!("proxy-core-{i}")) + .spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .or_panic("Failed to build Tokio runtime"); + let result = rt.block_on(async { + let mut listeners = Vec::with_capacity(std_listeners.len()); + for l in std_listeners { + listeners.push( + TcpListener::from_std(l) + .context("failed to register listener with tokio")?, + ); + } + let bal = balancer.zip(receiver); + accept_loop(listeners, app_state, None, bal).await + }); + if let Err(err) = result { + error!( + "proxy core {i} error on {}:{:?}: {err:?}", + config.listen_addr, config.listen_port + ); + } + }) + .with_context(|| format!("Failed to spawn proxy-core-{i} thread"))?; + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; + fn default_proxy_config() -> ProxyConfig { + crate::config::load_config_figment(None) + .focus("core.proxy") + .extract() + .expect("the shipped default config should parse") + } + + #[test] + fn ktls_probe_leaves_an_unconfigured_gateway_alone() { + let mut config = default_proxy_config(); + config.ktls = None; + disable_ktls_if_unsupported(&mut config); + assert!(config.ktls.is_none()); + } + + #[test] + fn ktls_survives_the_probe_only_when_the_kernel_supports_it() { + let mut config = default_proxy_config(); + config.ktls = Some(crate::config::EngageAfter::default()); + disable_ktls_if_unsupported(&mut config); + // Whichever way this kernel answers, the config must agree with it: + // keeping kTLS on a kernel without the ULP is what truncates responses + // at the gate. + assert_eq!(config.ktls.is_some(), probe_ktls().is_ok()); + } + #[test] fn test_parse_destination() { // Test basic app_id only diff --git a/dstack/gateway/src/proxy/adaptive_ktls.rs b/dstack/gateway/src/proxy/adaptive_ktls.rs new file mode 100644 index 000000000..c7b951585 --- /dev/null +++ b/dstack/gateway/src/proxy/adaptive_ktls.rs @@ -0,0 +1,294 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Traffic-triggered kernel TLS offload. +//! +//! Measurements show the two halves of kTLS pull in opposite directions: +//! enabling it costs ~30% of connection setup rate (secret extraction plus +//! kernel ULP setup per connection) but wins ~25% on bulk throughput once +//! combined with splice. Short request/response connections therefore pay the +//! setup cost and never earn it back. +//! +//! This module keeps the connection in userspace rustls after the handshake and +//! only hands it to the kernel once it has proven itself: once the configured +//! [`EngageAfter`] gate fires, the stream is drained at a TLS record boundary +//! and switched to kTLS + splice for the remainder. +//! +//! Handing over mid-stream is sound because the secrets rustls exports carry +//! the current record sequence numbers, and `CorkStream` exists precisely to +//! stop reads at a record boundary so nothing is left half-parsed. + +use std::time::{Duration, Instant}; + +use anyhow::{bail, Context, Result}; +use ktls::CorkStream; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::net::TcpStream; +use tokio_rustls::server::TlsStream; +use tracing::debug; + +use super::idle::IdleWatchdog; +use super::splice::{splice_bidirectional, CloseKind}; +use super::tls_terminate::SocketParts; +use crate::config::{EngageAfter, SpliceConfig}; + +/// Why the userspace relay phase stopped. +enum Phase { + /// The connection proved itself worth the offload. + Gated, + /// One side closed before the gate fired. + Eof, +} + +/// Relay both directions in userspace until either side closes or `gate` fires. +async fn relay_until<S>( + tls: &mut S, + upstream: &mut TcpStream, + gate: &EngageAfter, + idle: Option<Duration>, +) -> Result<Phase> +where + S: AsyncRead + AsyncWrite + Unpin, +{ + let (mut tr, mut tw) = tokio::io::split(tls); + let (mut ur, mut uw) = upstream.split(); + let mut down = vec![0u8; 32 * 1024]; + let mut up = vec![0u8; 32 * 1024]; + let mut moved: u64 = 0; + let start = Instant::now(); + let mut watchdog: Option<IdleWatchdog> = match idle { + Some(idle) => { + let mut w = IdleWatchdog::new(idle); + w.tick().await; // the first tick completes immediately + Some(w) + } + None => None, + }; + + let phase = loop { + // One side closing is not the end of the connection: a client that ends + // its request with close_notify still expects the response. Propagate + // the EOF to that direction's peer, then drain the other direction + // before giving up on the connection. + macro_rules! finish_one { + ($closing:expr, $r:expr, $w:expr, $buf:expr) => {{ + $closing.shutdown().await.ok(); + loop { + let n = tokio::select! { + () = async { match watchdog.as_mut() { + Some(w) => w.tick().await, + None => std::future::pending().await, + } } => { + // The drain is watched too: a backend that accepts the + // request and then never answers would otherwise hold + // the connection until `timeouts.total`. + if watchdog.as_mut().is_some_and(|w| w.stalled(moved)) { + bail!("idle timeout"); + } + continue; + } + r = $r.read(&mut $buf) => r.context("read error")?, + }; + if n == 0 { + break; + } + // Not `write_all`: it is not cancel-safe, so it cannot sit + // in a `select!`, and leaving it outside meant a client + // that stopped reading blocked the drain in its write with + // the watchdog unpolled -- the mirror of the silent-backend + // stall, and just as good for holding a connection to + // `timeouts.total`. Single `write` calls are cancel-safe + // (nothing is written when the other branch wins), so the + // partial-write loop is ours to drive. + let mut written = 0usize; + while written < n { + let count = tokio::select! { + () = async { match watchdog.as_mut() { + Some(w) => w.tick().await, + None => std::future::pending().await, + } } => { + if watchdog.as_mut().is_some_and(|w| w.stalled(moved)) { + bail!("idle timeout"); + } + continue; + } + r = $w.write(&$buf[written..n]) => r.context("write error")?, + }; + if count == 0 { + bail!("write accepted no bytes"); + } + written += count; + // Per partial write, so a peer draining slowly still + // counts as progress and is not reaped for being slow. + moved += count as u64; + } + } + $w.shutdown().await.ok(); + break Phase::Eof; + }}; + } + tokio::select! { + () = async { match watchdog.as_mut() { + Some(w) => w.tick().await, + // No idle timeout configured: this arm must never win. + None => std::future::pending().await, + } } => { + if watchdog.as_mut().is_some_and(|w| w.stalled(moved)) { + bail!("idle timeout"); + } + continue; + } + r = tr.read(&mut down) => { + let n = r.context("read from client failed")?; + if n == 0 { finish_one!(uw, ur, tw, up); } + uw.write_all(&down[..n]).await.context("write to app failed")?; + moved += n as u64; + } + r = ur.read(&mut up) => { + let n = r.context("read from app failed")?; + if n == 0 { finish_one!(tw, tr, uw, down); } + tw.write_all(&up[..n]).await.context("write to client failed")?; + moved += n as u64; + } + } + if gate.reached(moved, start) { + // Flush before handing the socket to the kernel so no plaintext is + // still sitting in a rustls write buffer. + tw.flush().await.context("flush before offload failed")?; + break Phase::Gated; + } + }; + Ok(phase) +} + +/// Relay a freshly accepted TLS connection, upgrading it to kTLS + splice once +/// the kTLS gate fires. `splice` supplies the relay settings used afterwards. +pub(crate) async fn relay_with_adaptive_offload<IO>( + mut tls: TlsStream<CorkStream<IO>>, + mut upstream: TcpStream, + ktls: &EngageAfter, + splice: &SpliceConfig, + idle: Option<Duration>, +) -> Result<()> +where + IO: AsyncRead + AsyncWrite + Unpin + std::os::fd::AsRawFd + ktls::AsyncReadReady, + IO: SocketParts, +{ + match relay_until(&mut tls, &mut upstream, ktls, idle).await? { + Phase::Eof => return Ok(()), + Phase::Gated => {} + } + debug!("offloading connection to kTLS after {ktls:?}"); + + // config_ktls_server corks the stream, drains rustls to a record boundary + // and installs the current traffic secrets into the kernel. + let ktls_stream = super::stats::record_ktls_offload(ktls::config_ktls_server(tls).await) + .context("failed to switch connection to kernel TLS")?; + let (drained, io) = ktls_stream.into_raw(); + if let Some(drained) = drained { + if !drained.is_empty() { + upstream + .write_all(&drained) + .await + .context("failed to flush drained data to app")?; + } + } + // Same rule as the immediate offload path: the sniff remainder is raw + // ciphertext, so it can neither be forwarded to the app nor pushed back + // into a socket the kernel now owns. A completed handshake always consumes + // it, so this is unreachable -- refuse rather than corrupt a stream if it + // ever is not. + let (buffered, tcp) = io.into_socket_parts(); + if !buffered.is_empty() { + bail!( + "{} bytes of unconsumed ciphertext at kTLS handover", + buffered.len() + ); + } + // The client side is now a kTLS socket, so its close needs a close_notify. + splice_bidirectional( + tcp, + upstream, + splice.release_idle_pipes, + idle, + CloseKind::KernelTls, + ) + .await + .context("splice after kTLS offload failed") +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::net::TcpListener; + + async fn connected_pair() -> (TcpStream, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let client = TcpStream::connect(addr).await.unwrap(); + let (server, _) = listener.accept().await.unwrap(); + (client, server) + } + + /// A gate no test connection reaches, so the relay stays in the userspace + /// phase for the whole exchange. `relay_until` is generic over the client + /// stream, so a plain socket stands in for the TLS one and the half-close + /// handling is exercised without a handshake. + fn ungated() -> EngageAfter { + EngageAfter { + after_bytes: Some(1 << 30), + after_duration: None, + } + } + + #[tokio::test] + async fn response_survives_a_client_half_close_before_the_gate() { + let (mut client, mut tls_side) = connected_pair().await; + let (mut upstream, mut backend) = connected_pair().await; + let relay = tokio::spawn(async move { + relay_until(&mut tls_side, &mut upstream, &ungated(), None).await + }); + + client.write_all(b"ping").await.unwrap(); + client.shutdown().await.unwrap(); + + let mut req = vec![0u8; 4]; + backend.read_exact(&mut req).await.unwrap(); + assert_eq!(&req, b"ping"); + let mut trailing = Vec::new(); + backend.read_to_end(&mut trailing).await.unwrap(); + assert!(trailing.is_empty()); + backend.write_all(b"pong").await.unwrap(); + drop(backend); + + let mut resp = Vec::new(); + client.read_to_end(&mut resp).await.unwrap(); + assert_eq!(resp, b"pong", "client lost the response after half-closing"); + assert!(matches!(relay.await.unwrap().unwrap(), Phase::Eof)); + } + + #[tokio::test] + async fn request_survives_an_app_half_close_before_the_gate() { + let (mut client, mut tls_side) = connected_pair().await; + let (mut upstream, mut backend) = connected_pair().await; + let relay = tokio::spawn(async move { + relay_until(&mut tls_side, &mut upstream, &ungated(), None).await + }); + + backend.write_all(b"early").await.unwrap(); + backend.shutdown().await.unwrap(); + + let mut resp = vec![0u8; 5]; + client.read_exact(&mut resp).await.unwrap(); + assert_eq!(&resp, b"early"); + + client.write_all(b"late").await.unwrap(); + drop(client); + + let mut got = Vec::new(); + backend.read_to_end(&mut got).await.unwrap(); + assert_eq!(got, b"late", "app lost the request after half-closing"); + assert!(matches!(relay.await.unwrap().unwrap(), Phase::Eof)); + } +} diff --git a/dstack/gateway/src/proxy/balance.rs b/dstack/gateway/src/proxy/balance.rs new file mode 100644 index 000000000..a065c4ae9 --- /dev/null +++ b/dstack/gateway/src/proxy/balance.rs @@ -0,0 +1,205 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Evening out connections across the thread-per-core workers. +//! +//! `SO_REUSEPORT` assigns a connection to a listener by hashing its 4-tuple, and +//! thread-per-core cannot move a connection afterwards, so a core that draws few +//! connections idles while its neighbours saturate. Measured at 16 connections +//! over 4 cores, utilisation came out like `[99, 42, 101, 101]` and throughput +//! varied 46% between restarts on nothing but how the hash fell. +//! +//! Steering the kernel's choice was tried first and failed (see +//! [`super::reuseport`]). This instead lets the accepting core hand the +//! connection to a less loaded one. The cost is a channel send per *rebalanced +//! connection* -- not per connection, and never per request -- so the +//! thread-per-core property that made this model fast in the first place is kept +//! for everything already in flight. +//! +//! This is the same shape as HAProxy's multi-queue accept, which picks the least +//! loaded of three candidate threads and pushes the connection onto its ring. +//! Two of its choices were tried here and did not transfer: capping accepts per +//! wakeup (its `maxaccept`) and routing every connection through the channel even +//! when it stays local both measured within noise. Its shared listening socket +//! measured clearly worse for us -- 227k against 245k at 16 connections -- since +//! every worker's reactor then wakes on every connection. Its own numbers say the +//! same thing from the other side: put HAProxy on per-thread reuseport listeners +//! (`shards by-thread`) and it drops 15%, below our thread-per-core. + +use std::net::SocketAddr; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use tokio::net::TcpStream; +use tokio::sync::mpsc; +use tracing::{debug, warn}; + +/// A connection accounted to one core, released when the connection ends. +pub(crate) struct CoreSlot { + counts: Arc<Vec<AtomicUsize>>, + core: usize, +} + +impl CoreSlot { + fn claim(counts: Arc<Vec<AtomicUsize>>, core: usize) -> Self { + counts[core].fetch_add(1, Ordering::Relaxed); + Self { counts, core } + } +} + +impl Drop for CoreSlot { + fn drop(&mut self) { + self.counts[self.core].fetch_sub(1, Ordering::Relaxed); + } +} + +/// A connection handed over from another core. +/// +/// Carried as a `std::net::TcpStream`, deliberately: a tokio `TcpStream` stays +/// registered with the reactor that accepted it, so moving the object to another +/// core would leave its readiness handling behind and cost a cross-core hop on +/// every read and write for the life of the connection. Handing over the plain +/// socket lets the target core register it with its own reactor. +pub(crate) type Handoff = (std::net::TcpStream, SocketAddr, CoreSlot); + +/// Per-core view of the shared connection counts and handoff channels. +pub(crate) struct Balancer { + counts: Arc<Vec<AtomicUsize>>, + senders: Arc<Vec<mpsc::Sender<Handoff>>>, + me: usize, +} + +/// Handoffs a core may have waiting before others stop offering it work. +/// +/// The queue is bounded on purpose. A core only receives connections while it +/// is the *least* loaded, so in steady state this never fills. What it protects +/// against is a core that stops draining -- wedged or gone: with an unbounded +/// queue the others keep succeeding at `send`, and every connection they hand +/// over sits unserved until `timeouts.total` while its `CoreSlot` keeps the +/// core looking busier, so the queue and the memory behind it only grow. Full +/// means "this core is not actually taking work", and the sender keeps the +/// connection instead. +const HANDOFF_QUEUE: usize = 1024; + +/// How far above the least loaded core this one has to be before handing a +/// connection over. +/// +/// Proportional, not fixed: a slack of 2 stops a core being starved at 4 +/// connections per core but keeps churning at 12, and every migration costs a +/// little locality. Measured against a fixed slack of 2 on passthrough +/// small-request: +6.5% at 16 connections, and the worst-loaded core goes from +/// 81% to 97% busy. +fn migration_threshold(least: usize) -> usize { + least + 1 + least / 4 +} + +impl Balancer { + /// Build one balancer per core, plus the receiver each core listens on. + pub(crate) fn build(workers: usize) -> (Vec<Self>, Vec<mpsc::Receiver<Handoff>>) { + let counts = Arc::new( + (0..workers) + .map(|_| AtomicUsize::new(0)) + .collect::<Vec<_>>(), + ); + let mut senders = Vec::with_capacity(workers); + let mut receivers = Vec::with_capacity(workers); + for _ in 0..workers { + let (tx, rx) = mpsc::channel(HANDOFF_QUEUE); + senders.push(tx); + receivers.push(rx); + } + let senders = Arc::new(senders); + let balancers = (0..workers) + .map(|me| Self { + counts: counts.clone(), + senders: senders.clone(), + me, + }) + .collect(); + (balancers, receivers) + } + + /// Which core should own a freshly accepted connection. + fn target(&self) -> usize { + if self.counts.len() < 2 { + return self.me; + } + let mine = self.counts[self.me].load(Ordering::Relaxed); + let mut best = self.me; + let mut best_val = mine; + for (i, c) in self.counts.iter().enumerate() { + let v = c.load(Ordering::Relaxed); + if v < best_val { + best = i; + best_val = v; + } + } + if mine >= migration_threshold(best_val) { + best + } else { + self.me + } + } + + /// Account a newly accepted connection, handing it to a less loaded core if + /// this one is running ahead. + /// + /// Returns the slot to keep alongside the connection when it stays here, or + /// `None` once the connection has been handed away. + /// + /// Handing over is best-effort: every failure path falls back to serving the + /// connection on this core, because a connection served by a busier core is + /// strictly better than one that is dropped. The one exception is + /// [`TcpStream::into_std`], which consumes the stream and closes the socket + /// when it fails -- there is nothing left to fall back to, so that case is + /// logged rather than silently counted as a handover. + pub(crate) fn place( + &self, + stream: TcpStream, + from: SocketAddr, + ) -> Option<(TcpStream, CoreSlot)> { + let target = self.target(); + if target == self.me { + let slot = CoreSlot::claim(self.counts.clone(), self.me); + return Some((stream, slot)); + } + // Drop this core's reactor registration before handing the socket over. + let raw = match stream.into_std() { + Ok(raw) => raw, + Err(err) => { + // `into_std` took ownership, so the socket is already closed. + warn!("dropping connection from {from}: failed to deregister for handover: {err}"); + return None; + } + }; + let slot = CoreSlot::claim(self.counts.clone(), target); + // `try_send`, not `send`: this runs on the accept path and must not wait + // on a core that is not draining. See `HANDOFF_QUEUE`. + let raw = match self.senders[target].try_send((raw, from, slot)) { + Ok(()) => return None, + Err(mpsc::error::TrySendError::Full((raw, _, _))) => { + debug!( + "core {target} handoff queue is full; keeping connection on core {}", + self.me + ); + raw + } + Err(mpsc::error::TrySendError::Closed((raw, _, _))) => { + debug!( + "core {target} is gone; keeping connection on core {}", + self.me + ); + raw + } + }; + match TcpStream::from_std(raw) { + Ok(stream) => Some((stream, CoreSlot::claim(self.counts.clone(), self.me))), + Err(err) => { + // Same as above: `from_std` consumed the socket. + warn!("dropping connection from {from}: failed to re-adopt after handover: {err}"); + None + } + } + } +} diff --git a/dstack/gateway/src/proxy/health_check.rs b/dstack/gateway/src/proxy/health_check.rs new file mode 100644 index 000000000..42c4023f4 --- /dev/null +++ b/dstack/gateway/src/proxy/health_check.rs @@ -0,0 +1,926 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Application-level health polling of registered CVMs. +//! +//! A CVM registers from `dstack-util setup`, during boot, before +//! `app-compose.service` has started a single container. From that moment it is +//! eligible for traffic, and the only thing standing between it and a request +//! is the WireGuard handshake age -- which says the tunnel is up and nothing +//! about whether anything is listening behind it. +//! +//! The gateway polls rather than having the CVM push. A guest agent that is +//! wedged, OOM-killed or mid-crash then shows up as a failed poll, instead of +//! as silence that has to be told apart from "nothing changed" with a +//! timestamp and a staleness heuristic. It also means "I could not reach it" +//! is a per-node observation, exactly like the WireGuard handshakes this sits +//! next to, rather than shared state that outlives whatever wrote it. +//! +//! Only instances that asked to be gated are polled -- an app opts in with +//! `requirements.health_check`, which reaches the gateway as +//! `RegisterCvmRequest.health_check`. That flag, not a probe, is what keeps an +//! older or uninterested CVM out of this: every failed poll counts as +//! unhealthy, so discovering support by trying would blackhole exactly the +//! instances that cannot answer. +//! +//! Reachability is plain HTTP over the WireGuard tunnel, the same way +//! [`super::port_policy`] already fetches port policy from the agent. + +use std::collections::{BTreeMap, BTreeSet}; +use std::net::Ipv4Addr; +use std::time::Duration; + +use dstack_guest_agent_rpc::v1::worker_client::WorkerClient as WorkerV1Client; +use futures::StreamExt; +use http_client::ConnectionReuse; +use tokio::time::MissedTickBehavior; +use tracing::{error, info, warn}; + +use crate::config::HealthCheckConfig; +use crate::main_service::Proxy; +use crate::models::HealthState; + +/// How long to wait before restarting the poll loop after it dies. +const RESTART_DELAY: Duration = Duration::from_secs(5); + +/// One poll's verdict, plus why, for the line logged when the verdict changes. +pub(crate) struct Observation { + pub state: HealthState, + pub reason: String, +} + +/// What one `Worker.Health` call produced. +/// +/// "The agent said no" and "the agent did not answer" are different claims and +/// are treated differently by [`apply_hysteresis`], so they stay apart until +/// something decides what to do with them. +#[derive(Debug, PartialEq, Eq)] +enum PollResult { + Healthy, + /// The agent answered, and the answer was no. Build with + /// [`PollResult::unhealthy`]; the reason is app-authored. + Unhealthy(String), + /// No answer: timed out, refused, unparseable. Build with + /// [`PollResult::unreachable`]; the reason quotes a transport error, which + /// is not the same as being free of guest-controlled bytes. + Unreachable(String), +} + +impl PollResult { + /// The agent answered "no", for `reason`. + fn unhealthy(reason: impl AsRef<str>) -> Self { + Self::Unhealthy(sanitize(reason.as_ref())) + } + + /// No usable answer arrived, because of `reason`. + /// + /// Sanitizing in the constructor rather than at each call site, because the + /// obvious-looking call sites are the dangerous ones. A decode failure + /// stringifies a `serde_json` error, and serde renders a type mismatch by + /// quoting the offending value in full: a CVM answering `{"healthy":"<16 + /// MiB>"}` turns into a 16 MiB log line, newlines and terminal escapes + /// intact. That is the same attack the answered-unhealthy path is bounded + /// against, arriving through the branch that reads like plumbing. + fn unreachable(reason: impl AsRef<str>) -> Self { + Self::Unreachable(sanitize(reason.as_ref())) + } +} + +/// One instance as the round snapshotted it. +/// +/// The WireGuard key and IP travel with the verdict so that +/// `record_instance_health` can tell whether the record it is about to write to +/// is still the one that was polled. A round takes as long as its slowest +/// instance, and a CVM can reboot and re-register inside that window. +#[derive(Clone, Debug)] +pub(crate) struct Target { + pub id: String, + pub ip: Ipv4Addr, + pub public_key: String, +} + +/// Start the polling loop. Called once at startup. +pub(crate) fn spawn_poller(state: Proxy) { + let config = state.config.proxy.health_check.clone(); + if !config.enabled { + info!("application health polling is disabled; every instance stays eligible"); + return; + } + info!( + "polling application health every {:?} (timeout {:?})", + config.interval, config.timeout + ); + tokio::spawn(async move { + // Supervised rather than a bare spawn. If the loop ever dies -- a + // poisoned `ProxyState` mutex is enough, since locking it panics -- + // polling would otherwise stop for the life of the process with nothing + // logged, and every instance registered after that point would sit at + // `Unknown` forever while the stale verdicts around it kept serving. + loop { + let task = tokio::spawn(poll_forever(state.clone(), config.clone())); + match task.await { + Ok(()) => error!("health poller returned unexpectedly; restarting"), + Err(err) => error!("health poller died: {err}; restarting"), + } + tokio::time::sleep(RESTART_DELAY).await; + } + }); +} + +async fn poll_forever(state: Proxy, config: HealthCheckConfig) { + let mut ticker = tokio::time::interval(config.interval); + // A round that overruns the interval must not leave catch-up rounds queued + // behind it; delay the next tick instead. + ticker.set_missed_tick_behavior(MissedTickBehavior::Delay); + // Consecutive failures to reach each agent, so a single dropped packet does + // not eject an instance. Rebuilt from the round's own targets each time, so + // it cannot outlive the instances it describes. + let mut failures = BTreeMap::<String, u32>::new(); + // Where the next round starts in the target list. See `poll_round`. + let mut cursor = 0usize; + loop { + ticker.tick().await; + cursor = poll_round(&state, &config, &mut failures, cursor).await; + } +} + +async fn poll_round( + state: &Proxy, + config: &HealthCheckConfig, + failures: &mut BTreeMap<String, u32>, + cursor: usize, +) -> usize { + // Snapshot under the lock and poll outside it. These are network round + // trips, and holding `ProxyState` across them would stall every connection + // being routed. + let mut targets = select_targets(state); + // Set membership rather than a scan per entry: both sides are the fleet, so + // the nested form is quadratic in it, re-run every interval. + let live = targets + .iter() + .map(|target| target.id.as_str()) + .collect::<BTreeSet<_>>(); + failures.retain(|id, _| live.contains(id.as_str())); + drop(live); + if targets.is_empty() { + return 0; + } + // Start where the last round stopped. The round is bounded below, so + // without rotating, the instances past the budget would never be polled at + // all -- and which instances those are is decided by `BTreeMap` ordering, + // i.e. by instance id, which a tenant chooses. + let total = targets.len(); + targets.rotate_left(cursor % total); + + let agent_port = state.config.proxy.agent_port; + let timeout = config.timeout; + let mut polls = futures::stream::iter(targets) + .map(|target| async move { + let result = poll_instance(target.ip, agent_port, timeout).await; + (target, result) + }) + .buffer_unordered(config.concurrency.max(1)); + + // The round gets one interval, no more. A poll that never answers costs a + // full `timeout` slot, so a fleet with enough of them takes + // `ceil(n / concurrency) * timeout` -- and `MissedTickBehavior::Delay` then + // makes that the *interval*, so one tenant's mute CVMs become every other + // tenant's detection latency. Measured on this transport: 256 such + // instances take a 33s round at the shipped defaults. + let deadline = tokio::time::Instant::now() + config.interval; + let mut polled = 0usize; + loop { + let landed = tokio::select! { + biased; + landed = polls.next() => landed, + _ = tokio::time::sleep_until(deadline) => break, + }; + let Some((target, result)) = landed else { + break; + }; + polled += 1; + // Each verdict is applied as its poll lands, not batched at the end of + // the round. A batch would hold the first instance's answer for as long + // as the slowest one takes and then write it onto whatever record + // exists by then -- which, for a CVM that rebooted in between, is a + // different boot of the app being told it is healthy on the strength of + // the previous one. + let Some(observation) = apply_hysteresis(result, &target, config, failures) else { + // Not enough evidence yet; the instance keeps the verdict it has. + continue; + }; + state.lock().record_instance_health(&target, observation); + } + if polled < total { + // Never silently. An operator whose fleet is not being fully polled has + // to be able to see that, and the number is the signal that something + // is absorbing the budget. + warn!( + "health round polled {polled} of {total} instances before its {:?} budget ran out; \ + the rest are first in line next round", + config.interval + ); + } + cursor.wrapping_add(polled) +} + +/// The instances worth polling this round. +pub(crate) fn select_targets(state: &Proxy) -> Vec<Target> { + // Reachability, from the same cache the routing path uses. A CVM that is + // powered off stays in `state.instances` for up to `recycle.timeout` (10h + // by default) while already being excluded from routing by handshake age; + // polling it would burn a full `timeout` slot every round, and enough of + // them stretch the round past the interval and slow health detection down + // for every live instance in the fleet. + let stale_after = state.config.proxy.timeouts.handshake_stale; + // Fetched before the lock, not under it. Building this map clones every + // peer's public key, and on a cold cache it shells out to `wg show` + // synchronously -- neither belongs inside the mutex the routing path takes + // once per connection. + let handshakes = state.latest_handshakes(None).unwrap_or_default(); + let guard = state.lock(); + guard + .state + .instances + .values() + // Instances that never asked to be polled cannot be gated on the + // answer, so polling them would only produce failures to misread. + .filter(|instance| instance.health_check()) + .filter(|instance| { + // No handshake recorded yet is not evidence of anything: a CVM that + // just registered has not completed one. Only a handshake that + // exists and has gone stale means "gone". + handshakes + .get(&instance.public_key) + .is_none_or(|(_, elapsed)| *elapsed < stale_after) + }) + .map(|instance| Target { + id: instance.id.clone(), + ip: instance.ip, + public_key: instance.public_key.clone(), + }) + .collect() +} + +/// Require several consecutive failures to *reach* an agent before demoting. +/// +/// An agent that answers "unhealthy" is believed immediately: it can see the +/// app and this gateway cannot. But a timed-out or refused poll is far more +/// often a transient than an outage, and acting on the first one ejects the +/// instance *and* recomputes the whole app's selection -- twice, once on the +/// way out and once on the way back. +/// +/// `None` means "no verdict this round": the instance keeps whatever it had, +/// which is the whole point. Writing a provisional state instead would demote +/// it on the first failure through the back door. +fn apply_hysteresis( + result: PollResult, + target: &Target, + config: &HealthCheckConfig, + failures: &mut BTreeMap<String, u32>, +) -> Option<Observation> { + let reason = match result { + PollResult::Healthy => { + failures.remove(&target.id); + return Some(Observation { + state: HealthState::Healthy, + reason: String::new(), + }); + } + PollResult::Unhealthy(reason) => { + failures.remove(&target.id); + return Some(Observation { + state: HealthState::Unhealthy, + reason, + }); + } + PollResult::Unreachable(reason) => reason, + }; + let seen = failures.entry(target.id.clone()).or_insert(0); + // Counted before the comparison, so a configured zero or one both mean + // "demote on the first failure" -- there is no way to demote before one has + // been observed. + *seen += 1; + if *seen < config.failure_threshold { + return None; + } + Some(Observation { + state: HealthState::Unhealthy, + reason: format!("{reason} ({seen} consecutive failures)"), + }) +} + +/// Ask one agent whether its app is serving. +/// +/// A failure to get an answer comes back as [`PollResult::Unreachable`] rather +/// than as a verdict; [`apply_hysteresis`] decides when enough of them in a row +/// amount to one. +async fn poll_instance(ip: Ipv4Addr, agent_port: u16, timeout: Duration) -> PollResult { + let client = WorkerV1Client::new(prober_transport(ip, agent_port)); + let response = match tokio::time::timeout(timeout, client.health(Default::default())).await { + Err(_) => { + return PollResult::unreachable(format!("health poll timed out after {timeout:?}")) + } + Ok(Err(err)) => return PollResult::unreachable(format!("health poll failed: {err:#}")), + Ok(Ok(response)) => response, + }; + if response.healthy { + return PollResult::Healthy; + } + PollResult::unhealthy(describe_unhealthy(&response)) +} + +/// How the poller talks to one agent. +/// +/// Named, rather than built inline, so the two decisions it makes can be +/// asserted. Both are the kind a refactor drops without anything failing. +/// +/// `guest_agent_client` bounds the response: `health_check` is self-declared by +/// the CVM and this runs on a timer against the whole fleet, so an unbounded +/// body here is an out-of-memory the guest gets to schedule. +/// +/// The connection is fresh rather than pooled, because opening it is half of +/// what this call asks. An agent out of file descriptors, or with a full accept +/// backlog, keeps answering whoever is already connected while refusing +/// everyone new -- and every connection the gateway proxies to the app is a new +/// one. A probe riding a pooled connection would report healthy for an instance +/// no real request can reach. Only the connection is unshared; the client +/// itself is process-wide. See `http_client::ConnectionReuse`. +fn prober_transport(ip: Ipv4Addr, agent_port: u16) -> http_client::prpc::PrpcClient { + // `/prpc/v1`, not `/prpc`: `Health` is a v1 `Worker` method. + // + // No released agent is affected. `Health` never shipped in a release, and a + // pre-0.6 gateway does not poll, so the only guests that can be asked are + // 0.6+ ones that opted in via `RegisterCvmRequest.health_check`. + // + // The one skew that does exist is unreleased: an interim `next` build that + // served `Health` at `/prpc` and registered with `health_check = true` will + // now 404 every poll, and a 404 counts as unreachable, so after + // `failure_threshold` polls the instance drops out of app-id rotation. + // Instance-id routing keeps working and a restart on a current build fixes + // it. Not worth a compatibility probe on every poll for a build nobody was + // asked to run. + let url = format!("http://{ip}:{agent_port}/prpc/v1"); + super::guest_agent_client(url, ConnectionReuse::Fresh) +} + +/// Longest reason kept from an agent's answer. +/// +/// This string ends up in a `info!` line. It is diagnostic only -- routing is +/// decided by `healthy` alone -- so there is nothing to lose by cutting it. +const MAX_REASON_BYTES: usize = 512; + +/// How many unhealthy entries are named before the rest are counted. +const MAX_NAMED_CONTAINERS: usize = 8; + +/// Summarize an agent's "no" for the log line. The caller sanitizes. +fn describe_unhealthy(response: &dstack_guest_agent_rpc::v1::HealthResponse) -> String { + if !response.error.is_empty() { + return format!("agent could not determine app health: {}", response.error); + } + if response.unhealthy.is_empty() { + return "agent reported unhealthy without naming anything".to_string(); + } + let mut described = response + .unhealthy + .iter() + .take(MAX_NAMED_CONTAINERS) + .map(|container| format!("{} is {}", container.name, container.status)) + .collect::<Vec<_>>() + .join(", "); + if let Some(rest) = response.unhealthy.len().checked_sub(MAX_NAMED_CONTAINERS) { + if rest > 0 { + described.push_str(&format!(" (and {rest} more)")); + } + } + described +} + +/// Make an agent's answer safe to log. +/// +/// The guest agent sanitizes what it puts in a report, but a guest agent is +/// exactly the party this gateway does not trust -- that assumption is why +/// health is polled rather than pushed. So the bytes are re-bounded and +/// stripped here, where the log line is actually emitted. Otherwise a hostile +/// CVM alternating healthy and unhealthy gets one attacker-chosen log line per +/// interval, newlines and escapes included. +/// +/// Called from [`PollResult::unhealthy`] and [`PollResult::unreachable`] rather +/// than from each site that formats a reason, because the site that was missed +/// was the one that looked like plumbing rather than like an answer. +fn sanitize(reason: &str) -> String { + dstack_types::sanitize_for_log(reason, MAX_REASON_BYTES) +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + /// The bound is a per-call-site opt-in now, not a property of the + /// transport, so nothing but a test stops a refactor from dropping it -- + /// and dropping it restores an out-of-memory a hostile guest gets to + /// schedule, since `health_check` is self-declared and this runs on a timer + /// against the whole fleet. + #[test] + fn every_guest_agent_client_bounds_the_response() { + let client = crate::proxy::guest_agent_client( + "http://10.0.0.2:8090/prpc".to_string(), + ConnectionReuse::Pooled, + ); + assert!( + client.max_response_bytes().is_some(), + "a guest agent is untrusted; its response must be bounded" + ); + } + + /// A stand-in for a CVM's guest agent, on loopback. + /// + /// `poll_instance` is handed an address and does everything else itself: + /// forms the pRPC request, reads the answer under a bound, and turns a + /// transport failure into `Unreachable` rather than into a verdict. None of + /// that is reachable from a unit test that injects a `PollResult`, and it + /// is the one hop between the gateway and an untrusted peer -- so it gets a + /// real socket. + /// + /// `answer` of `None` accepts the connection and then says nothing, which + /// is what a wedged agent looks like from here: not a refusal, which fails + /// fast, but silence that has to be timed out. + async fn fake_agent(answer: Option<Vec<u8>>) -> u16 { + fake_agent_capturing(answer, None).await + } + + /// The same stand-in, optionally reporting the request line it was sent. + /// + /// Kept as one implementation because the drain below is what stops this + /// fixture flaking: answering into a socket the peer is still writing to is + /// a reset on some platforms. + async fn fake_agent_capturing( + answer: Option<Vec<u8>>, + request_line: Option<tokio::sync::oneshot::Sender<String>>, + ) -> u16 { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .expect("bind"); + let port = listener.local_addr().expect("addr").port(); + tokio::spawn(async move { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + // Drain the request first. Answering into a socket the peer is + // still writing to is a reset on some platforms, which would show + // up as a flake rather than as the case under test. + let mut pending = Vec::new(); + let mut buf = [0u8; 1024]; + let headers_end = loop { + if let Some(at) = pending.windows(4).position(|w| w == b"\r\n\r\n") { + break at + 4; + } + match socket.read(&mut buf).await { + Ok(0) | Err(_) => return, + Ok(n) => pending.extend_from_slice(&buf[..n]), + } + }; + let content_length = String::from_utf8_lossy(&pending[..headers_end]) + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::<usize>().ok())? + }) + .unwrap_or(0); + while pending.len() < headers_end + content_length { + match socket.read(&mut buf).await { + Ok(0) | Err(_) => return, + Ok(n) => pending.extend_from_slice(&buf[..n]), + } + } + if let Some(tx) = request_line { + let head = String::from_utf8_lossy(&pending[..headers_end]).into_owned(); + let _ = tx.send(head.lines().next().unwrap_or_default().to_string()); + } + let Some(answer) = answer else { + std::future::pending::<()>().await; + return; + }; + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n", + answer.len() + ); + // Ignored: an oversized answer is refused mid-body, so the client + // is gone long before the last chunk goes out. That is the case + // under test, not a failure of the fixture. + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(&answer).await; + }); + port + } + + /// The poller must ask the versioned surface. `Health` lives on the v1 + /// `Worker` at `/prpc/v1`; the unversioned `Worker` at `/prpc` is closed at + /// v0.5.11 and has no such method, so a poller pointed there gets a 404 that + /// `apply_hysteresis` eventually turns into "unhealthy" for every instance + /// in the fleet at once. + #[tokio::test] + async fn the_poller_asks_the_v1_surface() { + let (tx, rx) = tokio::sync::oneshot::channel(); + let port = fake_agent_capturing(Some(br#"{"healthy":true}"#.to_vec()), Some(tx)).await; + + let _ = poll_instance(Ipv4Addr::LOCALHOST, port, Duration::from_secs(5)).await; + + let request_line = rx.await.expect("the poller sent no request"); + assert!( + request_line.contains("/prpc/v1/Health"), + "expected the v1 Health path, got: {request_line}" + ); + } + + async fn poll_fake(answer: Option<Vec<u8>>) -> PollResult { + let port = fake_agent(answer).await; + poll_instance(Ipv4Addr::LOCALHOST, port, Duration::from_secs(5)).await + } + + #[tokio::test] + async fn an_agent_that_says_yes_is_healthy() { + assert_eq!( + poll_fake(Some(br#"{"healthy":true}"#.to_vec())).await, + PollResult::Healthy + ); + } + + /// The answer has to survive being decoded, not just being received: the + /// container list is what an operator reads to find out which container is + /// holding the instance out of rotation. + #[tokio::test] + async fn an_agent_that_says_no_is_believed_and_names_what_failed() { + let answer = br#"{"healthy":false,"unhealthy":[{"name":"web","status":"unhealthy"}]}"#; + let PollResult::Unhealthy(reason) = poll_fake(Some(answer.to_vec())).await else { + panic!("an agent answering `no` must be believed on the spot"); + }; + assert!(reason.contains("web is unhealthy"), "unexpected: {reason}"); + } + + /// Silence is not a verdict. It has to arrive as `Unreachable`, because + /// that is the only thing `apply_hysteresis` will make an app wait + /// `failure_threshold` rounds for -- an `Unhealthy` here would demote on + /// the first dropped packet. + #[tokio::test] + async fn an_agent_that_accepts_and_then_says_nothing_times_out() { + let port = fake_agent(None).await; + let result = poll_instance(Ipv4Addr::LOCALHOST, port, Duration::from_millis(100)).await; + let PollResult::Unreachable(reason) = result else { + panic!("a silent agent must not produce a verdict, got {result:?}"); + }; + assert!(reason.contains("timed out"), "unexpected: {reason}"); + } + + #[tokio::test] + async fn nothing_listening_is_unreachable_rather_than_unhealthy() { + // Bound and dropped, so the port is one nothing answers on. + let port = { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .expect("bind"); + listener.local_addr().expect("addr").port() + }; + let result = poll_instance(Ipv4Addr::LOCALHOST, port, Duration::from_secs(5)).await; + assert!( + matches!(result, PollResult::Unreachable(_)), + "a refused connection must not read as an agent saying no, got {result:?}" + ); + } + + /// The bound the whole "a guest agent is untrusted" argument rests on, + /// asserted where it is actually applied rather than where it is set. + /// `every_guest_agent_client_bounds_the_response` proves the call site asks + /// for one and the transport has its own test that it enforces one; this is + /// the test that they are wired to each other. + #[tokio::test] + async fn an_oversized_answer_is_refused_instead_of_buffered() { + let mut answer = br#"{"healthy":true,"padding":""#.to_vec(); + answer.resize(crate::proxy::MAX_GUEST_RESPONSE_BYTES + 4096, b'A'); + answer.extend_from_slice(br#""}"#); + let result = poll_fake(Some(answer)).await; + assert!( + matches!(result, PollResult::Unreachable(_)), + "an unbounded answer is an out-of-memory the guest gets to schedule, got {result:?}" + ); + } + + /// A decode failure is the sanitizer's worst case and the one that was + /// missed once: serde renders a type mismatch by quoting the offending + /// value, so the guest picks the bytes. Covered in isolation already -- + /// this is the same thing arriving over a socket, which is how it would + /// actually happen. + #[tokio::test] + async fn a_hostile_answer_cannot_write_its_own_log_line() { + let answer = format!( + r#"{{"healthy":"{}"}}"#, + "\\u000a[ERROR] fabricated\\u0000".repeat(64) + ); + let result = poll_fake(Some(answer.into_bytes())).await; + let PollResult::Unreachable(reason) = result else { + panic!("an undecodable answer is not a verdict, got {result:?}"); + }; + assert!( + !reason.contains('\n') && !reason.contains('\u{0}'), + "control characters reached a log line: {reason:?}" + ); + // The marker truncation appends is allowed past the bound, the same + // way `an_unreachable_reason_is_bounded_and_stripped_too` has it. + const MARKER: &str = "... (truncated)"; + assert!( + reason.len() <= MAX_REASON_BYTES + MARKER.len(), + "reason was not bounded: {} bytes", + reason.len() + ); + // Asserted, not assumed: without it this test would also pass on a + // reason that was simply short, which proves nothing about the bound. + assert!(reason.ends_with(MARKER), "not truncated: {reason:?}"); + } + + /// The probe must open its own connection. Reuse is the cheaper default + /// and would be adopted by anyone tidying this up, so the reason it is + /// wrong here is written down in `prober_transport` and asserted here: a + /// pooled connection reports on an agent's ability to serve a conversation + /// it is already in, while the traffic this gates is every time a *new* + /// one. An agent that has run out of descriptors passes the first and + /// fails the second. + #[test] + fn the_probe_does_not_ride_a_connection_someone_else_opened() { + assert_eq!( + prober_transport(Ipv4Addr::new(10, 0, 0, 2), 8090).connection_reuse(), + ConnectionReuse::Fresh, + "a probe over a pooled connection cannot see an agent that refuses new ones" + ); + } + + fn config(failure_threshold: u32) -> HealthCheckConfig { + HealthCheckConfig { + enabled: true, + interval: Duration::from_secs(5), + timeout: Duration::from_secs(2), + concurrency: 16, + failure_threshold, + } + } + + fn target() -> Target { + Target { + id: "inst-1".to_string(), + ip: Ipv4Addr::new(10, 0, 0, 2), + public_key: "key-1".to_string(), + } + } + + fn unreachable() -> PollResult { + PollResult::unreachable("health poll timed out after 2s") + } + + /// The reason exactly as it reaches a log line. + /// + /// `describe_unhealthy` composes it and the constructor bounds it, and only + /// the pair is ever what an operator sees -- asserting on `describe_unhealthy` + /// alone would pass with the sanitizing dropped. + fn unhealthy_reason(response: &dstack_guest_agent_rpc::v1::HealthResponse) -> String { + match PollResult::unhealthy(describe_unhealthy(response)) { + PollResult::Unhealthy(reason) => reason, + other => panic!("expected an unhealthy verdict, got {other:?}"), + } + } + + /// One dropped packet must not eject an instance and recompute the app's + /// whole selection. + #[test] + fn a_single_failure_does_not_demote() { + let mut failures = BTreeMap::new(); + assert!( + apply_hysteresis(unreachable(), &target(), &config(2), &mut failures).is_none(), + "one dropped poll must leave the current verdict alone" + ); + } + + #[test] + fn consecutive_failures_reach_the_threshold_and_demote() { + let mut failures = BTreeMap::new(); + let config = config(2); + apply_hysteresis(unreachable(), &target(), &config, &mut failures); + let observation = apply_hysteresis(unreachable(), &target(), &config, &mut failures) + .expect("second failure should demote"); + assert_eq!(observation.state, HealthState::Unhealthy); + assert!(observation.reason.contains("2 consecutive failures")); + } + + #[test] + fn one_successful_poll_clears_the_streak() { + let mut failures = BTreeMap::new(); + let config = config(2); + apply_hysteresis(unreachable(), &target(), &config, &mut failures); + apply_hysteresis(PollResult::Healthy, &target(), &config, &mut failures); + assert!(failures.is_empty(), "streak survived a healthy poll"); + assert!( + apply_hysteresis(unreachable(), &target(), &config, &mut failures).is_none(), + "the next failure must start counting from zero" + ); + } + + /// The agent can see the app and this gateway cannot, so its own "no" is + /// not something to sit on. + #[test] + fn an_agent_reporting_unhealthy_is_believed_immediately() { + let mut failures = BTreeMap::new(); + let observation = apply_hysteresis( + PollResult::Unhealthy("web is starting".to_string()), + &target(), + &config(3), + &mut failures, + ) + .expect("an answered poll is a verdict"); + assert_eq!(observation.state, HealthState::Unhealthy); + assert_eq!(observation.reason, "web is starting"); + } + + /// An answer also clears a streak, so an app that flaps between "cannot + /// reach" and "reachable but unhealthy" still demotes on the answer. + #[test] + fn an_answered_poll_clears_the_streak_too() { + let mut failures = BTreeMap::new(); + let config = config(3); + apply_hysteresis(unreachable(), &target(), &config, &mut failures); + apply_hysteresis( + PollResult::Unhealthy("db is unhealthy".to_string()), + &target(), + &config, + &mut failures, + ); + assert!(failures.is_empty()); + } + + /// Turning hysteresis off must demote on the first failure -- and cannot do + /// anything sooner, since a failure is counted before it is compared. + #[test] + fn a_threshold_below_one_demotes_on_the_first_failure() { + for threshold in [0, 1] { + let mut failures = BTreeMap::new(); + let observation = + apply_hysteresis(unreachable(), &target(), &config(threshold), &mut failures) + .unwrap_or_else(|| panic!("threshold {threshold} should demote")); + assert_eq!(observation.state, HealthState::Unhealthy); + assert!(observation.reason.contains("1 consecutive failures")); + } + } + + #[test] + fn an_agent_error_is_described_rather_than_dropped() { + let response = dstack_guest_agent_rpc::v1::HealthResponse { + healthy: false, + unhealthy: vec![], + error: "failed to connect to docker".to_string(), + }; + assert!(describe_unhealthy(&response).contains("failed to connect to docker")); + } + + #[test] + fn unhealthy_containers_are_named() { + let response = dstack_guest_agent_rpc::v1::HealthResponse { + healthy: false, + unhealthy: vec![ + dstack_guest_agent_rpc::v1::ContainerHealth { + name: "web".to_string(), + status: "starting".to_string(), + }, + dstack_guest_agent_rpc::v1::ContainerHealth { + name: "db".to_string(), + status: "unhealthy".to_string(), + }, + ], + error: String::new(), + }; + assert_eq!( + describe_unhealthy(&response), + "web is starting, db is unhealthy" + ); + } + + /// The guest agent sanitizes its own report, but a guest agent is exactly + /// the party this gateway does not trust. A hostile one must not be able to + /// forge a log line or repaint a terminal. + #[test] + fn control_characters_from_an_agent_never_reach_a_log_line() { + let response = dstack_guest_agent_rpc::v1::HealthResponse { + healthy: false, + unhealthy: vec![dstack_guest_agent_rpc::v1::ContainerHealth { + name: "web\n2026-01-01 INFO forged".to_string(), + status: "\x1b[31mstarting\r".to_string(), + }], + error: String::new(), + }; + let described = unhealthy_reason(&response); + assert!(!described.contains('\n'), "newline survived: {described:?}"); + assert!(!described.contains('\r'), "CR survived: {described:?}"); + assert!( + !described.contains('\x1b'), + "escape survived: {described:?}" + ); + } + + /// `char::is_control` alone lets these through, and both change what a + /// reader sees: U+2028 is a line break to most log viewers, U+202E reverses + /// the rendering of everything after it. + #[test] + fn line_separators_and_bidi_overrides_are_stripped_too() { + let response = dstack_guest_agent_rpc::v1::HealthResponse { + healthy: false, + unhealthy: vec![], + error: "web\u{2028}Jan 01 INFO ok\u{202E}desrever".to_string(), + }; + let described = unhealthy_reason(&response); + assert!(!described.contains('\u{2028}'), "{described:?}"); + assert!(!described.contains('\u{202E}'), "{described:?}"); + } + + /// The only bound on the response body is a megabyte, and this line is + /// emitted every time the verdict flips. + #[test] + fn an_oversized_reason_is_truncated() { + let response = dstack_guest_agent_rpc::v1::HealthResponse { + healthy: false, + unhealthy: vec![], + error: "x".repeat(64 * 1024), + }; + let described = unhealthy_reason(&response); + assert!( + described.len() < MAX_REASON_BYTES * 2, + "reason was {} bytes", + described.len() + ); + assert!(described.ends_with("(truncated)")); + } + + /// The other half of the same rule, on the branch that reads like plumbing. + /// + /// An unreachable reason stringifies whatever failed, and a decode failure + /// stringifies a `serde_json` error -- which renders a type mismatch by + /// quoting the offending value *in full*. A CVM answering + /// `{"healthy":"<16 MiB of text>"}` therefore emitted a 16 MiB log line + /// with its newlines and terminal escapes intact, and could schedule one + /// every interval by alternating healthy and garbage. The answered-unhealthy + /// path was bounded from the start; this one was not, because nothing about + /// a transport error looks guest-controlled. + #[test] + fn an_unreachable_reason_is_bounded_and_stripped_too() { + // The marker the truncation appends, which is allowed past the bound. + const MARKER: &str = "... (truncated)"; + let quoted = "\nlevel=fatal msg=owned\u{1b}[31m".repeat(4096); + let hostile = format!("health poll failed: invalid type: string \"{quoted}\""); + assert!( + hostile.len() > MAX_REASON_BYTES * 4, + "the fixture must exceed the bound, or this test proves nothing" + ); + + let PollResult::Unreachable(reason) = PollResult::unreachable(&hostile) else { + panic!("unreachable() must build an Unreachable"); + }; + assert!( + reason.len() <= MAX_REASON_BYTES + MARKER.len(), + "reason was {} bytes", + reason.len() + ); + assert!(reason.ends_with(MARKER), "not truncated: {reason:?}"); + assert!(!reason.contains('\n'), "newline survived: {reason:?}"); + assert!(!reason.contains('\u{1b}'), "escape survived: {reason:?}"); + } + + /// A thousand containers must not become a thousand-entry log line. + #[test] + fn only_the_first_few_containers_are_named() { + let response = dstack_guest_agent_rpc::v1::HealthResponse { + healthy: false, + unhealthy: (0..50) + .map(|index| dstack_guest_agent_rpc::v1::ContainerHealth { + name: format!("svc-{index}"), + status: "starting".to_string(), + }) + .collect(), + error: String::new(), + }; + let described = describe_unhealthy(&response); + assert!(described.contains("svc-0")); + assert!(!described.contains("svc-40")); + assert!(described.contains("and 42 more"), "got: {described}"); + } + + /// An agent that says "not healthy" but names nothing must still produce a + /// reason an operator can act on. + #[test] + fn an_unhealthy_response_naming_nothing_still_has_a_reason() { + let response = dstack_guest_agent_rpc::v1::HealthResponse { + healthy: false, + unhealthy: vec![], + error: String::new(), + }; + assert!(!describe_unhealthy(&response).is_empty()); + } +} diff --git a/dstack/gateway/src/proxy/idle.rs b/dstack/gateway/src/proxy/idle.rs new file mode 100644 index 000000000..34ce04f64 --- /dev/null +++ b/dstack/gateway/src/proxy/idle.rs @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! One idle watchdog for every relay path. +//! +//! `timeouts.idle` used to be enforced per read, which meant a connection died +//! when *one* direction went quiet even while the other was busy. It is now a +//! connection-level watchdog that samples a monotonic progress counter: if +//! neither direction has moved for the idle window, the connection is stalled. +//! That costs one timer per window rather than one per operation, which is why +//! the fast paths can afford it. +//! +//! It lives here rather than inside a relay because every relay needs it and +//! they are shaped differently: the buffered bridge and the pre-gate relays are +//! `select!` loops that can poll it as one more branch, while a spliced +//! connection has no loop to hang it on and races it against the transfer +//! instead. Both call the same sampling logic, so the three paths cannot drift +//! into enforcing different things -- which is exactly what happened when +//! splice and kTLS silently had no idle timeout at all. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use tokio::time::{interval, Interval, MissedTickBehavior}; + +/// Samples a progress counter a few times per idle window. +pub(crate) struct IdleWatchdog { + ticker: Interval, + /// Consecutive ticks that saw no progress. + idle_ticks: u32, + /// Ticks without progress that add up to the configured idle window. + max_idle_ticks: u32, + last_seen: u64, +} + +impl IdleWatchdog { + pub(crate) fn new(idle: Duration) -> Self { + // Four samples per window bounds the overshoot at 25% while keeping the + // timer rate proportional to the window rather than to the traffic. The + // floor stops a tiny `idle` turning into a busy loop. + let tick = (idle / 4).max(Duration::from_millis(500)); + let mut ticker = interval(tick); + ticker.set_missed_tick_behavior(MissedTickBehavior::Delay); + Self { + ticker, + idle_ticks: 0, + max_idle_ticks: (idle.as_millis() / tick.as_millis()).max(1) as u32, + last_seen: 0, + } + } + + /// Wait for the next sample point. + /// + /// Cancel-safe: `Interval::tick` is, and nothing else is held across it, so + /// this can sit in a `select!` arm that loses the race. + pub(crate) async fn tick(&mut self) { + self.ticker.tick().await; + } + + /// Record where the relay has got to. `true` means it has been stalled for + /// the whole idle window. + pub(crate) fn stalled(&mut self, progress: u64) -> bool { + if progress == self.last_seen { + self.idle_ticks += 1; + self.idle_ticks >= self.max_idle_ticks + } else { + self.idle_ticks = 0; + self.last_seen = progress; + false + } + } + + /// Resolve once the relay has been idle for the whole window. + /// + /// For relays with no loop of their own to poll from -- a spliced + /// connection is two joined transfers, not a `select!` -- so this is raced + /// against the transfer instead. + pub(crate) async fn wait_until_stalled(mut self, progress: &AtomicU64) { + // The first tick completes immediately; consume it so the first real + // sample is a full tick away. + self.tick().await; + loop { + self.tick().await; + if self.stalled(progress.load(Ordering::Relaxed)) { + return; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + #[tokio::test(start_paused = true)] + async fn a_stalled_relay_is_reported_after_the_window() { + let progress = Arc::new(AtomicU64::new(0)); + let watchdog = IdleWatchdog::new(Duration::from_secs(4)); + let start = tokio::time::Instant::now(); + watchdog.wait_until_stalled(&progress).await; + // Sampling four times per window means it fires within one tick of the + // window, never before it. + let waited = start.elapsed(); + assert!( + waited >= Duration::from_secs(4) && waited <= Duration::from_secs(6), + "fired after {waited:?}" + ); + } + + #[tokio::test(start_paused = true)] + async fn a_relay_that_keeps_moving_is_never_reported() { + let progress = Arc::new(AtomicU64::new(0)); + let bump = progress.clone(); + tokio::spawn(async move { + for _ in 0..20 { + tokio::time::sleep(Duration::from_secs(1)).await; + bump.fetch_add(1, Ordering::Relaxed); + } + }); + let watchdog = IdleWatchdog::new(Duration::from_secs(4)); + let fired = tokio::time::timeout( + Duration::from_secs(15), + watchdog.wait_until_stalled(&progress), + ) + .await; + assert!( + fired.is_err(), + "watchdog fired on a connection that was moving" + ); + } +} diff --git a/dstack/gateway/src/proxy/io_bridge.rs b/dstack/gateway/src/proxy/io_bridge.rs index 8c5dd7662..70cda0a77 100644 --- a/dstack/gateway/src/proxy/io_bridge.rs +++ b/dstack/gateway/src/proxy/io_bridge.rs @@ -2,10 +2,12 @@ // // SPDX-License-Identifier: Apache-2.0 +use super::idle::IdleWatchdog; use crate::config::ProxyConfig; -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use bytes::BytesMut; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::net::TcpStream; use tokio::time::timeout; use tracing::{debug, trace}; @@ -25,6 +27,10 @@ struct OneDirection<'a, R, W> { reader: &'a mut R, writer: &'a mut W, next_step: NextStep, + /// Bumped whenever this direction makes progress. The watchdog samples it + /// instead of the clock, so the hot path costs an integer increment rather + /// than arming a timer per read/write/flush. + progress: u64, } impl<R, W> OneDirection<'_, R, W> @@ -35,11 +41,12 @@ where async fn step(&mut self) -> Result<bool> { match self.next_step { NextStep::Read => { - let n = timeout(self.cfg.timeouts.idle, self.reader.read_buf(&mut self.buf)) + let n = self + .reader + .read_buf(&mut self.buf) .await - .ok() - .context("idle timeout")? .context("read error")?; + self.progress += 1; trace!(direction = %self.dir, "read: {n} bytes"); if n == 0 { self.next_step = NextStep::Shutdown; @@ -49,25 +56,19 @@ where Ok(false) } NextStep::Write => { - timeout( - self.cfg.timeouts.write, - self.writer.write_buf(&mut self.buf), - ) - .await - .ok() - .context("write timeout")? - .context("write error")?; + self.writer + .write_buf(&mut self.buf) + .await + .context("write error")?; + self.progress += 1; if self.buf.is_empty() { self.next_step = NextStep::Flush; } Ok(false) } NextStep::Flush => { - timeout(self.cfg.timeouts.write, self.writer.flush()) - .await - .ok() - .context("flush timeout")? - .context("flush error")?; + self.writer.flush().await.context("flush error")?; + self.progress += 1; self.next_step = NextStep::Read; Ok(false) } @@ -90,6 +91,30 @@ enum Rest<A, B> { B2a(B), } +/// Relay between two TCP sockets. +/// +/// Same logic as [`bridge`], but splitting a `TcpStream` with its own `split()` +/// hands out borrowed halves, where the generic `tokio::io::split` has to wrap +/// the stream in a `BiLock`. That lock showed up at 1.1% of total time on the +/// small-request passthrough profile, and passthrough always has a plain socket +/// on both sides, so it never needs the generic path. +pub(crate) async fn bridge_tcp( + mut a: TcpStream, + mut b: TcpStream, + config: &ProxyConfig, +) -> Result<()> { + let buf_size = config.buffer_size; + if !config.timeouts.data_timeout_enabled { + tokio::io::copy_bidirectional_with_sizes(&mut a, &mut b, buf_size, buf_size) + .await + .context("failed to copy")?; + return Ok(()); + } + let (mut ra, mut wa) = a.split(); + let (mut rb, mut wb) = b.split(); + relay(&mut ra, &mut wa, &mut rb, &mut wb, config).await +} + pub(crate) async fn bridge<A, B>(mut a: A, mut b: B, config: &ProxyConfig) -> Result<()> where A: AsyncRead + AsyncWrite + Unpin, @@ -106,31 +131,67 @@ where let (mut ra, mut wa) = tokio::io::split(a); let (mut rb, mut wb) = tokio::io::split(b); + relay(&mut ra, &mut wa, &mut rb, &mut wb, config).await +} +/// Drive both directions until each has seen EOF and been shut down. +async fn relay<RA, WA, RB, WB>( + ra: &mut RA, + wa: &mut WA, + rb: &mut RB, + wb: &mut WB, + config: &ProxyConfig, +) -> Result<()> +where + RA: AsyncRead + Unpin, + WA: AsyncWrite + Unpin, + RB: AsyncRead + Unpin, + WB: AsyncWrite + Unpin, +{ + let buf_size = config.buffer_size; let mut a2b = OneDirection { dir: "a2b", cfg: config, buf: BytesMut::with_capacity(buf_size), - reader: &mut ra, - writer: &mut wb, + reader: ra, + writer: wb, next_step: NextStep::Read, + progress: 0, }; let mut b2a = OneDirection { dir: "b2a", cfg: config, buf: BytesMut::with_capacity(buf_size), - reader: &mut rb, - writer: &mut wa, + reader: rb, + writer: wa, next_step: NextStep::Read, + progress: 0, }; + // One watchdog for the whole connection replaces the per-operation timeouts: + // it samples both directions' progress counters, so a connection only dies + // when neither has moved. See `super::idle`. + let mut watchdog = IdleWatchdog::new(config.timeouts.idle); + watchdog.tick().await; // the first tick completes immediately + let mut rest; + // Progress of the direction that finishes first, frozen at that point. The + // watchdog samples a monotonic counter, so the drain phase has to keep + // adding it rather than restart from the surviving direction alone. + // Assigned on every path that leaves the loop, like `rest`. + let finished: u64; // Transfer data between a and b bidirectionally. loop { tokio::select! { + _ = watchdog.tick() => { + if watchdog.stalled(a2b.progress + b2a.progress) { + bail!("idle timeout"); + } + } done = a2b.step() => { if done? { // a to b is EOF, switch to b to a only + finished = a2b.progress; rest = Rest::B2a(b2a); drop(a2b); break; @@ -139,6 +200,7 @@ where done = b2a.step() => { if done? { // b to a is EOF, switch to a to b only + finished = b2a.progress; rest = Rest::A2b(a2b); drop(b2a); break; @@ -147,18 +209,39 @@ where } } - // One of the direction is closed, copy the other direction. + // One direction is closed; drain the other -- still watched. Half-close is + // not a licence to hang: before the watchdog existed each read carried its + // own `idle` timeout, so this phase was covered, and leaving it bare let a + // client hold a connection open until `timeouts.total` (5h) by + // half-closing against a backend that never replies. match &mut rest { - Rest::A2b(a2b) => loop { - if a2b.step().await? { - break; + Rest::A2b(a2b) => drain(a2b, &mut watchdog, finished).await, + Rest::B2a(b2a) => drain(b2a, &mut watchdog, finished).await, + } +} + +/// Pump the surviving direction to EOF, giving up if it stalls for `idle`. +async fn drain<R, W>( + dir: &mut OneDirection<'_, R, W>, + watchdog: &mut IdleWatchdog, + finished: u64, +) -> Result<()> +where + R: AsyncRead + Unpin, + W: AsyncWrite + Unpin, +{ + loop { + tokio::select! { + _ = watchdog.tick() => { + if watchdog.stalled(finished + dir.progress) { + bail!("idle timeout"); + } } - }, - Rest::B2a(b2a) => loop { - if b2a.step().await? { - break; + done = dir.step() => { + if done? { + return Ok(()); + } } - }, + } } - Ok(()) } diff --git a/dstack/gateway/src/proxy/port_policy.rs b/dstack/gateway/src/proxy/port_policy.rs index f2fd123ae..bc8ce05d1 100644 --- a/dstack/gateway/src/proxy/port_policy.rs +++ b/dstack/gateway/src/proxy/port_policy.rs @@ -18,9 +18,9 @@ use std::net::Ipv4Addr; use std::sync::{Arc, Mutex}; use anyhow::{bail, Context, Result}; -use dstack_guest_agent_rpc::dstack_guest_client::DstackGuestClient; +use dstack_guest_agent_rpc::v0::dstack_guest_client::DstackGuestClient; use dstack_types::AppCompose; -use http_client::prpc::PrpcClient; +use http_client::ConnectionReuse; use or_panic::ResultOrPanic; use tokio::sync::mpsc::UnboundedReceiver; use tracing::{debug, warn}; @@ -32,6 +32,7 @@ use crate::{ }; /// Outcome of a single fetch attempt, distinguishing what we can usefully retry. +#[derive(Debug)] enum FetchError { /// Transient: connection failed, RPC timed out, agent returned 5xx, etc. /// The CVM might just be warming up — retry with backoff. @@ -99,6 +100,13 @@ pub(crate) fn filter_allowed_addresses( port: u16, ) -> Result<AddressGroup> { let total = addresses.len(); + // Nothing was offered, so nothing was denied here -- and `port` has not + // been looked at yet, so naming it would send whoever reads this log at + // the wrong subsystem entirely. Selection knows why it came back empty; + // ask it instead of guessing. + if total == 0 { + bail!("{}", state.lock().describe_empty_selection(app_id)); + } let allowed: AddressGroup = addresses .into_iter() .filter(|a| match is_port_allowed(state, &a.instance_id, port) { @@ -226,7 +234,11 @@ async fn fetch_and_store(state: &Proxy, instance_id: &str) -> Result<(), FetchEr async fn fetch_port_policy(ip: Ipv4Addr, agent_port: u16) -> Result<PortPolicy, FetchError> { let url = format!("http://{ip}:{agent_port}/prpc"); - let client = DstackGuestClient::new(PrpcClient::new(url)); + // Same untrusted peer as the health poller; see `guest_agent_client`. + // Pooled, unlike the poller: this fills a cache rather than deciding + // anything, it fires once per instance instead of on a timer, and a + // connection that died while idle costs one retry of a lazy fetch. + let client = DstackGuestClient::new(super::guest_agent_client(url, ConnectionReuse::Pooled)); // Network/RPC errors here are transient: agent might still be coming up. let info = client .info() @@ -234,21 +246,22 @@ async fn fetch_port_policy(ip: Ipv4Addr, agent_port: u16) -> Result<PortPolicy, .context("agent Info() rpc failed") .map_err(FetchError::Transient)?; - // Anything below this point is the agent telling us something we can't - // turn into port_policy — treat as permanent. - if info.tcb_info.is_empty() { - // Legacy CVM with public_tcbinfo=false; we cannot inspect app-compose - // remotely. Cache the default (open) policy so we don't keep retrying. - // Apps that need restrict_mode must run a CVM that reports policy at - // registration time — they cannot rely on lazy fetch. + parse_info_port_policy(&info.tcb_info) +} + +fn parse_info_port_policy(tcb_info: &str) -> Result<PortPolicy, FetchError> { + // Legacy CVM with public_tcbinfo=false; we cannot inspect app-compose + // remotely. Cache the default (open) policy so we don't keep retrying. + // Apps that need restrict_mode must report policy during registration. + if tcb_info.is_empty() { return Ok(PortPolicy::default()); } - let tcb: serde_json::Value = serde_json::from_str(&info.tcb_info) + let tcb: serde_json::Value = serde_json::from_str(tcb_info) .context("invalid tcb_info json") .map_err(FetchError::Permanent)?; let raw = tcb .get("app_compose") - .and_then(|v| v.as_str()) + .and_then(|value| value.as_str()) .ok_or_else(|| FetchError::Permanent(anyhow::anyhow!("tcb_info missing app_compose")))?; let app_compose: AppCompose = serde_json::from_str(raw) .context("failed to parse app_compose from tcb_info") @@ -257,10 +270,58 @@ async fn fetch_port_policy(ip: Ipv4Addr, agent_port: u16) -> Result<PortPolicy, .port_policy .ports .into_iter() - .map(|p| (p.port, PortFlags { pp: p.pp })) + .map(|port| (port.port, PortFlags { pp: port.pp })) .collect(); Ok(PortPolicy { ports, restrict_mode: app_compose.port_policy.restrict_mode, }) } + +#[cfg(test)] +mod tests { + use super::{parse_info_port_policy, FetchError}; + + #[test] + fn legacy_empty_info_uses_bounded_open_compatibility_policy() { + let policy = parse_info_port_policy("").expect("legacy empty info rejected"); + assert!(!policy.restrict_mode); + assert!(policy.ports.is_empty()); + } + + #[test] + fn reported_policy_wins_and_preserves_proxy_protocol_flags() { + let compose = serde_json::json!({ + "manifest_version": "3", + "name": "port-policy-fixture", + "runner": "docker-compose", + "gateway_enabled": true, + "port_policy": { + "restrict_mode": true, + "ports": [ + {"port": 443, "pp": true}, + {"port": 8080, "pp": false} + ] + } + }); + let tcb = serde_json::json!({"app_compose": compose.to_string()}).to_string(); + let policy = parse_info_port_policy(&tcb).expect("reported policy rejected"); + assert!(policy.restrict_mode); + assert!(policy.ports.get(&443).expect("443 missing").pp); + assert!(!policy.ports.get(&8080).expect("8080 missing").pp); + } + + #[test] + fn malformed_or_missing_compose_is_permanent_failure() { + for value in [ + "not-json", + r#"{"other":"value"}"#, + r#"{"app_compose":"bad"}"#, + ] { + assert!(matches!( + parse_info_port_policy(value), + Err(FetchError::Permanent(_)) + )); + } + } +} diff --git a/dstack/gateway/src/proxy/reuseport.rs b/dstack/gateway/src/proxy/reuseport.rs new file mode 100644 index 000000000..b2be7a93d --- /dev/null +++ b/dstack/gateway/src/proxy/reuseport.rs @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! `SO_REUSEPORT` listener groups for the thread-per-core proxy. +//! +//! By default the kernel picks a listener from a reuseport group by hashing the +//! connection's 4-tuple. With a moderate number of long-lived connections that +//! distribution is visibly uneven, and because thread-per-core cannot migrate a +//! connection between runtimes, the cores that drew fewer connections sit idle +//! while the others saturate. Measured at 16 connections over 4 cores, per-core +//! utilisation came out like `[99, 42, 101, 101]` and throughput swung 46% +//! between restarts purely on how the hash fell. +//! +//! `SO_ATTACH_REUSEPORT_CBPF` steering was tried as a fix and measured worse: +//! classic BPF has no maps, so round-robin is not expressible, and the one +//! signal it does expose -- the CPU handling the packet -- collapses the +//! distribution rather than evening it out. Connections are established in a +//! burst, so only the one or two client CPUs active at that moment are sampled, +//! and `cpu % n` sent every connection to the same one or two listeners: 2.3 of +//! 4 cores active and 167k rps against 245k for the plain hash. A real fix needs +//! per-connection state, i.e. an eBPF program with a counter map. +//! +//! What is kept here is binding the group in one place, in order, which keeps +//! listener order deterministic and the setup out of the per-thread path. + +use std::net::{SocketAddr, TcpListener}; + +use anyhow::{bail, Context, Result}; + +/// Bind `count` `SO_REUSEPORT` listeners on `addr`, in order. +/// +/// Returns one listener per worker, in the same order the kernel indexes them. +pub(crate) fn bind_group(addr: SocketAddr, count: usize, backlog: i32) -> Result<Vec<TcpListener>> { + if count == 0 { + bail!("reuseport group needs at least one listener"); + } + let mut listeners = Vec::with_capacity(count); + for _ in 0..count { + let socket = socket2::Socket::new( + socket2::Domain::IPV4, + socket2::Type::STREAM, + Some(socket2::Protocol::TCP), + ) + .context("failed to create listening socket")?; + socket + .set_reuse_port(true) + .context("failed to set SO_REUSEPORT")?; + socket.set_reuse_address(true).ok(); + socket.set_nonblocking(true).ok(); + socket + .bind(&addr.into()) + .with_context(|| format!("failed to bind {addr}"))?; + socket.listen(backlog).context("failed to listen")?; + listeners.push(TcpListener::from(socket)); + } + Ok(listeners) +} diff --git a/dstack/gateway/src/proxy/splice.rs b/dstack/gateway/src/proxy/splice.rs new file mode 100644 index 000000000..6cf7b538f --- /dev/null +++ b/dstack/gateway/src/proxy/splice.rs @@ -0,0 +1,769 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Zero-copy TCP relay using `splice(2)`. +//! +//! For the TLS-passthrough path both sides of the proxy are raw `TcpStream`s +//! and the gateway never inspects the (encrypted) payload. Instead of copying +//! bytes through a userspace buffer we move them kernel-side through a pipe with +//! `splice(2)`, which avoids two copies per direction and the associated CPU. +//! +//! Only used on Linux and only for the passthrough path; TLS-terminate still +//! uses the buffered bridge because one side is a decrypted rustls stream. + +use std::os::fd::{AsRawFd, OwnedFd}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::{bail, Context, Result}; +use nix::fcntl::{fcntl, splice, FcntlArg, SpliceFFlags}; +use nix::sys::socket::{shutdown, Shutdown}; +use nix::unistd::pipe; +use or_panic::OptionOrPanic; +use tokio::io::{AsyncReadExt, AsyncWriteExt, Interest}; +use tokio::net::TcpStream; + +use super::idle::IdleWatchdog; +use crate::config::{EngageAfter, SpliceConfig}; + +/// How a relay endpoint's write side has to be closed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CloseKind { + /// Plain socket: a FIN says everything there is to say. + Tcp, + /// Kernel TLS: the peer also needs a `close_notify` alert, or it cannot + /// tell an orderly close from a truncated stream. + KernelTls, +} + +/// Bytes moved per `splice` syscall. Also the target pipe capacity so a full +/// read can be buffered kernel-side before draining to the destination. +const PIPE_CAPACITY: usize = 1 << 20; // 1 MiB + +fn set_pipe_capacity(fd: &OwnedFd, size: usize) { + // Best-effort: larger pipes mean fewer syscalls for bulk transfers. If the + // kernel rejects the size (e.g. over /proc/sys/fs/pipe-max-size) we simply + // keep the default capacity. + let _ = fcntl( + fd.as_raw_fd(), + FcntlArg::F_SETPIPE_SZ(size as std::os::raw::c_int), + ); +} + +fn errno_to_io(e: nix::errno::Errno) -> std::io::Error { + std::io::Error::from_raw_os_error(e as i32) +} + +/// Per-thread cache of splice pipes. +/// +/// Creating a pipe per direction per connection costs two `pipe2` calls plus +/// four descriptor closes, which is pure overhead for short connections: it +/// measurably raised passthrough connection-setup CPU (134 -> 142 us per +/// connection) while HAProxy, which pools its pipes, went the other way. +/// Reusing them keeps the bulk-transfer win without paying setup per +/// connection. Thread-local, so no locking -- and with thread-per-core a +/// connection stays on the thread that took the pipe. +/// Per thread. Each pooled pipe holds two descriptors, so the cap costs +/// `PIPE_POOL_MAX * 2 * workers` file descriptors at steady state -- 512 for the +/// 4-worker bench, ~4096 for a 32-worker gateway. That is fine given +/// `set_ulimit` raises RLIMIT_NOFILE to the hard limit, but it is why the number +/// is not larger. A 40-minute soak confirmed the pool fills to the cap and then +/// stops (fds 487 -> 543 -> flat, RSS flat at ~44 MB). +/// +/// With `SpliceConfig::release_idle_pipes` this cap stops being just a cache +/// size and becomes the actual descriptor bound: idle connections park their +/// pipes here instead of holding them, so steady-state use is +/// `2 * PIPE_POOL_MAX * workers` plus the pipes carrying data, rather than +/// `4 * connections`. Note that parking does not close anything -- the saving +/// comes from connections sharing a bounded set of pipes, not from idle +/// connections costing zero. +const PIPE_POOL_MAX: usize = 64; + +thread_local! { + static PIPE_POOL: std::cell::RefCell<Vec<(OwnedFd, OwnedFd)>> = + const { std::cell::RefCell::new(Vec::new()) }; + /// Pipes currently checked out of the pool. Test-only bookkeeping: together + /// with `PIPE_POOL.len()` it gives this thread's live pipe count exactly, + /// where counting `/proc/self/fd` would race with tests on other threads. + #[cfg(test)] + static PIPES_BORROWED: std::cell::Cell<usize> = const { std::cell::Cell::new(0) }; +} + +/// A splice pipe borrowed from the thread-local pool. +/// +/// Returned to the pool on drop, but only if the transfer drained it: a pipe +/// still holding bytes would corrupt the next connection that used it. +struct PooledPipe { + rd: Option<OwnedFd>, + wr: Option<OwnedFd>, + drained: bool, +} + +impl PooledPipe { + fn get() -> Result<Self> { + #[cfg(test)] + PIPES_BORROWED.with(|n| n.set(n.get() + 1)); + if let Some((rd, wr)) = PIPE_POOL.with(|p| p.borrow_mut().pop()) { + return Ok(Self { + rd: Some(rd), + wr: Some(wr), + drained: true, + }); + } + let (rd, wr) = pipe().context("failed to create splice pipe")?; + set_pipe_capacity(&wr, PIPE_CAPACITY); + Ok(Self { + rd: Some(rd), + wr: Some(wr), + drained: true, + }) + } + + /// The ends are `Option` only so `Drop` can move them into the pool, and + /// `Drop` is the last thing that runs, so both are always present here. + fn rd(&self) -> &OwnedFd { + self.rd.as_ref().or_panic("pipe read end present") + } + + fn wr(&self) -> &OwnedFd { + self.wr.as_ref().or_panic("pipe write end present") + } +} + +impl Drop for PooledPipe { + fn drop(&mut self) { + let (Some(rd), Some(wr)) = (self.rd.take(), self.wr.take()) else { + return; + }; + #[cfg(test)] + PIPES_BORROWED.with(|n| n.set(n.get() - 1)); + if !self.drained { + // Unknown residue: close instead of poisoning the pool. + return; + } + PIPE_POOL.with(|p| { + let mut pool = p.borrow_mut(); + if pool.len() < PIPE_POOL_MAX { + pool.push((rd, wr)); + } + }); + } +} + +/// Copy one direction (`src` -> `dst`) with splice until EOF, then half-close +/// the destination's write side. +/// +/// With `release_idle_pipes` the pipe is handed back to the pool whenever the +/// source runs dry, so a connection only pins descriptors while it actually has +/// bytes in flight. See `SpliceConfig::release_idle_pipes` for why that is safe +/// and what it costs. +async fn splice_one( + src: Arc<TcpStream>, + dst: Arc<TcpStream>, + release_idle_pipes: bool, + progress: &AtomicU64, + src_kind: CloseKind, + dst_close: CloseKind, +) -> Result<()> { + let mut pipe = PooledPipe::get()?; + + loop { + // Move a chunk from the source socket into the pipe. + // Try the syscall first and only wait for readiness when it actually + // blocks. `try_io` decides with one atomic read of the cached readiness + // flag, where `readable()` builds, polls and drops a `Readiness` future + // every time -- which showed up as ~2.4% of total time on the + // small-request passthrough profile, paid once per splice. + let n = loop { + match src.try_io(Interest::READABLE, || { + splice( + src.as_ref(), + None, + pipe.wr(), + None, + PIPE_CAPACITY, + SpliceFFlags::SPLICE_F_MOVE | SpliceFFlags::SPLICE_F_NONBLOCK, + ) + .map_err(errno_to_io) + }) { + Ok(n) => break n, + Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => { + // The pipe is empty on every path that reaches here: it was + // either just taken from the pool, or the drain below ran to + // completion, and a `WouldBlock` from the fill above added + // nothing. So it can be parked while the source is idle. + if release_idle_pipes { + drop(pipe); + src.readable().await.context("readable error")?; + pipe = PooledPipe::get()?; + } else { + src.readable().await.context("readable error")?; + } + } + // A kTLS socket refuses to splice a record that is not + // application data, and reports it as EINVAL. For a byte relay + // that is the end of the data stream, not a failure: it is how + // the client's close_notify arrives once the socket belongs to + // the kernel. Treating it as fatal took down the *other* + // direction too, so a client closing its request discarded the + // response the app had already written. + Err(ref e) + if matches!(src_kind, CloseKind::KernelTls) + && e.raw_os_error() == Some(libc::EINVAL) => + { + break 0; + } + Err(e) => return Err(e).context("splice src->pipe failed"), + } + }; + if n == 0 { + break; // EOF on source; the pipe was left empty by the last drain + } + // One relaxed increment per chunk -- not per byte, and dwarfed by the + // splice syscall it accompanies -- is what lets the idle watchdog see + // this connection without a timer per operation. + progress.fetch_add(1, Ordering::Relaxed); + // A chunk is in the pipe now: not safe to recycle until fully drained. + pipe.drained = false; + + // Drain the pipe fully into the destination socket. + let mut left = n; + while left > 0 { + match dst.try_io(Interest::WRITABLE, || { + splice( + pipe.rd(), + None, + dst.as_ref(), + None, + left, + SpliceFFlags::SPLICE_F_MOVE | SpliceFFlags::SPLICE_F_NONBLOCK, + ) + .map_err(errno_to_io) + }) { + Ok(m) => { + left -= m; + if left == 0 { + // Fully drained: the pipe is empty again, so it can go + // back to the pool even if the connection dies next. + pipe.drained = true; + } + } + Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => { + dst.writable().await.context("writable error")?; + } + Err(e) => return Err(e).context("splice pipe->dst failed"), + } + } + } + + // Propagate EOF: half-close the write side so the peer sees the close. + // + // A kTLS socket needs the TLS-level close first. `KtlsStream::poll_shutdown` + // would have sent it, but the offload path hands the bare descriptor to + // splice and never goes through that type again, so a plain shutdown here + // reaches the client as a FIN with no close_notify -- indistinguishable from + // a truncation attack to a client that checks. + if matches!(dst_close, CloseKind::KernelTls) { + let _ = ktls::send_close_notify(dst.as_raw_fd()); + } + let _ = shutdown(dst.as_raw_fd(), Shutdown::Write); + Ok(()) +} + +/// Bidirectional zero-copy relay between two TCP streams. +/// +/// `a` is the client side, `b` the app side; `a_close` describes what kind of +/// socket `a` is, which decides both how its write side is closed and how a +/// non-data record from it is interpreted. +/// `idle` is `None` when data timeouts are disabled. +pub(crate) async fn splice_bidirectional( + a: TcpStream, + b: TcpStream, + release_idle_pipes: bool, + idle: Option<Duration>, + a_close: CloseKind, +) -> Result<()> { + // The single funnel for zero-copy relaying, so counting here covers both the + // passthrough gate and the post-kTLS handover. + super::stats::record_splice_engaged(); + let a = Arc::new(a); + let b = Arc::new(b); + let progress = AtomicU64::new(0); + let relay = async { + // `a` is the client side: it is the one that may be a kTLS socket, so + // it is the source kind for a2b and the destination kind for b2a. + let a2b = splice_one( + a.clone(), + b.clone(), + release_idle_pipes, + &progress, + a_close, + CloseKind::Tcp, + ); + let b2a = splice_one(b, a, release_idle_pipes, &progress, CloseKind::Tcp, a_close); + tokio::try_join!(a2b, b2a)?; + Ok(()) + }; + let Some(idle) = idle else { + return relay.await; + }; + // Spliced bytes never enter this process, so there is no read to hang a + // timeout on; the watchdog races the transfer instead. + tokio::select! { + result = relay => result, + () = IdleWatchdog::new(idle).wait_until_stalled(&progress) => bail!("idle timeout"), + } +} + +/// Per-thread cache of relay buffers, for the same reason as the pipe pool: +/// a pair of `buffer_size` allocations per connection is a real cost when the +/// connection only carries a few hundred bytes. +const BUF_POOL_MAX: usize = 32; +/// Phase 1 only runs until the splice threshold, so it does not need the full +/// `buffer_size` used for bulk copying. +const RELAY_BUF_SIZE: usize = 16 * 1024; + +thread_local! { + static BUF_POOL: std::cell::RefCell<Vec<(Vec<u8>, Vec<u8>)>> = + const { std::cell::RefCell::new(Vec::new()) }; +} + +struct PooledBufs { + a: Vec<u8>, + b: Vec<u8>, +} + +impl PooledBufs { + fn get(buf_size: usize) -> Self { + let size = buf_size.clamp(4096, RELAY_BUF_SIZE); + if let Some((a, b)) = BUF_POOL.with(|p| p.borrow_mut().pop()) { + return Self { a, b }; + } + Self { + a: vec![0u8; size], + b: vec![0u8; size], + } + } +} + +impl Drop for PooledBufs { + fn drop(&mut self) { + let a = std::mem::take(&mut self.a); + let b = std::mem::take(&mut self.b); + BUF_POOL.with(|p| { + let mut pool = p.borrow_mut(); + if pool.len() < BUF_POOL_MAX { + pool.push((a, b)); + } + }); + } +} + +/// Relay both directions with plain reads/writes until `gate` is reached, then +/// report whether splice should take over. +/// +/// Returns `true` if the gate was reached and both sockets are still open, +/// `false` if the connection finished first (in which case it is fully done). +/// +/// The gate is only tested at the tail of the loop, which is the one point +/// where both directions are quiescent: the `select!` arms each finish their +/// `write_all` before falling through, so nothing is buffered in userspace and +/// the sockets can be handed to the kernel safely. A timer arm inside the +/// `select!` would also be safe, but it would promote connections that are +/// merely idle -- allocating a pipe for a stream with nothing to move -- so +/// checking the clock on activity is both cheaper and better behaved. +async fn relay_until( + a: &mut TcpStream, + b: &mut TcpStream, + gate: &EngageAfter, + buf_size: usize, + idle: Option<Duration>, +) -> Result<bool> { + let (mut ar, mut aw) = a.split(); + let (mut br, mut bw) = b.split(); + // Buffers come from a thread-local pool: allocating two of them per + // connection cost more than the pipe setup this phase exists to avoid. + let mut bufs = PooledBufs::get(buf_size); + let mut moved: u64 = 0; + let start = Instant::now(); + // `moved` doubles as the progress counter: it only advances when a transfer + // happened, which is exactly what the watchdog samples for. + let mut watchdog: Option<IdleWatchdog> = match idle { + Some(idle) => { + let mut w = IdleWatchdog::new(idle); + w.tick().await; // the first tick completes immediately + Some(w) + } + None => None, + }; + + loop { + // `finish_one` drains a half-closed connection without splice: once one + // side is done there is no long-lived stream left to optimise. + // + // Half-close is not end-of-connection. A client that finishes its + // request with `shutdown(SHUT_WR)` still expects the response, so the + // EOF is propagated to the *peer of the direction that ended* + // (`$closing`) and the opposite direction is pumped to completion. + // Shutting down the writer we are about to pump into instead would + // deliver the peer's EOF and drop everything still in flight. + macro_rules! finish_one { + ($closing:expr, $r:expr, $w:expr, $buf:expr) => {{ + $closing.shutdown().await.ok(); + loop { + let n = tokio::select! { + () = async { match watchdog.as_mut() { + Some(w) => w.tick().await, + None => std::future::pending().await, + } } => { + // The drain is watched too: a backend that accepts the + // request and then never answers would otherwise hold + // the connection until `timeouts.total`. + if watchdog.as_mut().is_some_and(|w| w.stalled(moved)) { + bail!("idle timeout"); + } + continue; + } + r = $r.read(&mut $buf) => r.context("read error")?, + }; + if n == 0 { + break; + } + // Not `write_all`: it is not cancel-safe, so it cannot sit + // in a `select!`, and leaving it outside meant a client + // that stopped reading blocked the drain in its write with + // the watchdog unpolled -- the mirror of the silent-backend + // stall, and just as good for holding a connection to + // `timeouts.total`. Single `write` calls are cancel-safe + // (nothing is written when the other branch wins), so the + // partial-write loop is ours to drive. + let mut written = 0usize; + while written < n { + let count = tokio::select! { + () = async { match watchdog.as_mut() { + Some(w) => w.tick().await, + None => std::future::pending().await, + } } => { + if watchdog.as_mut().is_some_and(|w| w.stalled(moved)) { + bail!("idle timeout"); + } + continue; + } + r = $w.write(&$buf[written..n]) => r.context("write error")?, + }; + if count == 0 { + bail!("write accepted no bytes"); + } + written += count; + // Per partial write, so a peer draining slowly still + // counts as progress and is not reaped for being slow. + moved += count as u64; + } + } + // Both directions are drained now; let the other peer see EOF. + $w.shutdown().await.ok(); + return Ok(false); + }}; + } + tokio::select! { + () = async { match watchdog.as_mut() { + Some(w) => w.tick().await, + // No idle timeout configured: this arm must never win. + None => std::future::pending().await, + } } => { + if watchdog.as_mut().is_some_and(|w| w.stalled(moved)) { + bail!("idle timeout"); + } + continue; + } + r = ar.read(&mut bufs.a) => { + let n = r.context("read from client failed")?; + // Client is done sending: tell the app, keep relaying its reply. + if n == 0 { finish_one!(bw, br, aw, bufs.b); } + bw.write_all(&bufs.a[..n]).await.context("write to app failed")?; + moved += n as u64; + } + r = br.read(&mut bufs.b) => { + let n = r.context("read from app failed")?; + // App is done replying: tell the client, keep relaying its input. + if n == 0 { finish_one!(aw, ar, bw, bufs.a); } + aw.write_all(&bufs.b[..n]).await.context("write to client failed")?; + moved += n as u64; + } + } + if gate.reached(moved, start) { + return Ok(true); + } + } +} + +/// Bidirectional relay that only switches to splice once the connection has +/// proven itself worth the syscalls. +/// +/// splice moves ~17 syscalls per connection to shift a small response (fill the +/// pipe, drain the pipe, plus readiness retries), where a read/write pair needs +/// two. Its benefit is per byte, its cost is per connection -- the same shape as +/// kTLS. Short request/response connections therefore never touch a pipe, while +/// connections that trip either gate still get zero-copy. +pub(crate) async fn splice_bidirectional_after( + mut a: TcpStream, + mut b: TcpStream, + config: &SpliceConfig, + buf_size: usize, + idle: Option<Duration>, +) -> Result<()> { + if relay_until(&mut a, &mut b, &config.engage, buf_size, idle).await? { + splice_bidirectional(a, b, config.release_idle_pipes, idle, CloseKind::Tcp).await + } else { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + use tokio::net::TcpListener; + + fn pool_len() -> usize { + PIPE_POOL.with(|pool| pool.borrow().len()) + } + + /// Descriptors this thread currently holds on a pipe: two per live pipe, + /// whether the pipe is parked in the pool or checked out by a relay. + /// + /// This is the number capacity planning cares about, so the tests assert on + /// it directly rather than trusting the pool alone as a proxy. + fn pipe_fds() -> usize { + 2 * (pool_len() + PIPES_BORROWED.with(|n| n.get())) + } + + async fn connected_pair() -> (TcpStream, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let client = TcpStream::connect(addr).await.unwrap(); + let (server, _) = listener.accept().await.unwrap(); + (client, server) + } + + /// Wire up `client <-> relay <-> backend` and start the relay. + /// + /// The relay runs on the same thread as the test (a `#[tokio::test]` + /// runtime is single-threaded), so it shares the thread-local pipe pool and + /// the test can observe what the relay parks there. + async fn start_relay(release_idle_pipes: bool) -> (TcpStream, TcpStream) { + let (client, inbound) = connected_pair().await; + let (outbound, backend) = connected_pair().await; + tokio::spawn(splice_bidirectional( + inbound, + outbound, + release_idle_pipes, + None, + CloseKind::Tcp, + )); + (client, backend) + } + + fn clear_pool() { + PIPE_POOL.with(|pool| pool.borrow_mut().clear()); + } + + /// Let the relay reach its next idle wait. + async fn settle() { + tokio::time::sleep(Duration::from_millis(50)).await; + } + + async fn expect(stream: &mut TcpStream, want: &[u8]) { + let mut got = vec![0u8; want.len()]; + stream.read_exact(&mut got).await.unwrap(); + assert_eq!(got, want); + } + + #[tokio::test] + async fn an_idle_relay_owns_no_pipe_when_release_is_on() { + clear_pool(); + let (mut client, mut backend) = start_relay(true).await; + client.write_all(b"ping").await.unwrap(); + expect(&mut backend, b"ping").await; + settle().await; + + // Only one pipe is ever created: each direction hands its pipe back + // before the other one asks for it, so an idle bidirectional relay + // converges on a single pipe shared through the pool -- and owns none + // of its own. + assert_eq!(pool_len(), 1, "the relay parked everything it borrowed"); + } + + #[tokio::test] + async fn an_idle_relay_pins_two_pipes_when_release_is_off() { + clear_pool(); + let (mut client, mut backend) = start_relay(false).await; + client.write_all(b"ping").await.unwrap(); + expect(&mut backend, b"ping").await; + settle().await; + + assert_eq!(pool_len(), 0, "both pipes stay pinned to the connection"); + } + + /// The headline claim: with release on, descriptor use tracks pipes that + /// are actually carrying data, not open connections. + /// + /// Note what this does *not* say. Released pipes stay open in the pool, so + /// the saving is not "idle connections cost nothing" but "idle connections + /// share": the steady-state bound moves from `4 * connections` to + /// `2 * PIPE_POOL_MAX * workers` plus whatever is in flight. + #[tokio::test] + async fn idle_connections_share_pipes_instead_of_each_pinning_four() { + clear_pool(); + assert_eq!(pipe_fds(), 0, "test starts with no pipes on this thread"); + let mut ends = Vec::new(); + for _ in 0..8 { + let (mut client, mut backend) = start_relay(true).await; + client.write_all(b"ping").await.unwrap(); + expect(&mut backend, b"ping").await; + // Without this the previous relay has not yet been polled back to + // its idle wait, so it still owns its pipe when the next one asks + // for one -- which is the in-flight case, not the idle case. + settle().await; + ends.push((client, backend)); + } + + // Held for the connection's lifetime this would be 8 * 4 = 32. + assert_eq!(pipe_fds(), 2, "8 idle connections share one pooled pipe"); + drop(ends); + } + + #[tokio::test] + async fn idle_connections_each_cost_four_descriptors_when_release_is_off() { + clear_pool(); + assert_eq!(pipe_fds(), 0, "test starts with no pipes on this thread"); + let mut ends = Vec::new(); + for _ in 0..8 { + let (mut client, mut backend) = start_relay(false).await; + client.write_all(b"ping").await.unwrap(); + expect(&mut backend, b"ping").await; + ends.push((client, backend)); + } + settle().await; + + assert_eq!(pipe_fds(), 8 * 4, "two pipes pinned per relay"); + drop(ends); + } + + #[tokio::test] + async fn data_survives_repeated_park_and_reacquire() { + clear_pool(); + let (mut client, mut backend) = start_relay(true).await; + + // Each gap forces the relay to park its pipe and take a fresh one from + // the pool, which is where a stale or half-drained pipe would corrupt + // the stream. + for i in 0..5u8 { + let up = [b'a' + i; 16]; + client.write_all(&up).await.unwrap(); + expect(&mut backend, &up).await; + settle().await; + + let down = [b'A' + i; 16]; + backend.write_all(&down).await.unwrap(); + expect(&mut client, &down).await; + settle().await; + } + + assert_eq!(pool_len(), 1); + } + + #[tokio::test] + async fn a_payload_larger_than_one_splice_still_arrives_intact() { + let (mut client, mut backend) = start_relay(true).await; + let payload: Vec<u8> = (0..512 * 1024).map(|i| (i % 251) as u8).collect(); + let sender = tokio::spawn({ + let payload = payload.clone(); + async move { + client.write_all(&payload).await.unwrap(); + client + } + }); + let mut got = vec![0u8; payload.len()]; + backend.read_exact(&mut got).await.unwrap(); + assert_eq!(got, payload); + drop(sender.await.unwrap()); + } + + #[tokio::test] + async fn eof_propagates_with_release_enabled() { + let (client, mut backend) = start_relay(true).await; + drop(client); + let mut got = Vec::new(); + backend.read_to_end(&mut got).await.unwrap(); + assert!(got.is_empty()); + } + + /// A gate that no test connection will ever reach, so the relay stays in + /// the pre-splice phase for the whole exchange. + fn ungated() -> SpliceConfig { + SpliceConfig { + engage: EngageAfter { + after_bytes: Some(1 << 30), + after_duration: None, + }, + release_idle_pipes: false, + } + } + + async fn start_gated_relay() -> (TcpStream, TcpStream) { + let (client, inbound) = connected_pair().await; + let (outbound, backend) = connected_pair().await; + tokio::spawn(async move { + splice_bidirectional_after(inbound, outbound, &ungated(), 16 * 1024, None).await + }); + (client, backend) + } + + /// Half-closing a request must not cost the response: the client shuts down + /// its write side, and the backend replies afterwards. + #[tokio::test] + async fn response_survives_a_client_half_close_before_the_gate() { + let (mut client, mut backend) = start_gated_relay().await; + + client.write_all(b"ping").await.unwrap(); + client.shutdown().await.unwrap(); + + let mut req = vec![0u8; 4]; + backend.read_exact(&mut req).await.unwrap(); + assert_eq!(&req, b"ping"); + // The app sees the client's EOF but is still free to answer. + let mut trailing = Vec::new(); + backend.read_to_end(&mut trailing).await.unwrap(); + assert!(trailing.is_empty()); + backend.write_all(b"pong").await.unwrap(); + drop(backend); + + let mut resp = Vec::new(); + client.read_to_end(&mut resp).await.unwrap(); + assert_eq!(resp, b"pong", "client lost the response after half-closing"); + } + + /// The mirror image: the backend finishes first and the client is still + /// sending. Its remaining bytes have to reach the app. + #[tokio::test] + async fn request_survives_a_backend_half_close_before_the_gate() { + let (mut client, mut backend) = start_gated_relay().await; + + backend.write_all(b"early").await.unwrap(); + backend.shutdown().await.unwrap(); + + let mut resp = vec![0u8; 5]; + client.read_exact(&mut resp).await.unwrap(); + assert_eq!(&resp, b"early"); + + client.write_all(b"late").await.unwrap(); + drop(client); + + let mut got = Vec::new(); + backend.read_to_end(&mut got).await.unwrap(); + assert_eq!(got, b"late", "app lost the request after half-closing"); + } +} diff --git a/dstack/gateway/src/proxy/stats.rs b/dstack/gateway/src/proxy/stats.rs new file mode 100644 index 000000000..f468e6412 --- /dev/null +++ b/dstack/gateway/src/proxy/stats.rs @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! What the data path is actually doing with kTLS and splice. +//! +//! Neither option is simply on or off. kTLS can be cleared at startup by the +//! capability probe, and both engage per connection only once their gate fires, +//! so the configured value does not tell an operator what is running -- and the +//! difference is worth 2-4x in per-connection memory. Two things are reported: +//! the effective mode, which answers "is it on at all", and per-connection +//! counters, which answer "is it engaging". +//! +//! All counters are monotonic since process start and are read without +//! synchronisation, so a snapshot can be marginally inconsistent between +//! fields. That is the right trade for numbers whose purpose is to be watched +//! over time; the alternative costs an ordering on the connection path. + +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +use dstack_gateway_rpc::ProxyAccelStatus; + +use crate::config::{EngageAfter, ProxyConfig}; + +/// Connections handed to the kernel's TLS ULP. +static KTLS_OFFLOADED: AtomicU64 = AtomicU64::new(0); +/// Connections where that handover failed. Non-zero here means connections are +/// being dropped or truncated at the gate, not merely running unaccelerated. +static KTLS_OFFLOAD_FAILED: AtomicU64 = AtomicU64::new(0); +/// Connections that entered a zero-copy splice relay. Counted on both paths -- +/// TLS passthrough, and terminate once kTLS has handed the socket over -- since +/// from the relay's point of view they are the same thing. +static SPLICE_ENGAGED: AtomicU64 = AtomicU64::new(0); +/// Set when the startup probe finds no TLS ULP. Without it a probe-disabled +/// gateway is indistinguishable from one that was never configured for kTLS, +/// because both end up with `ProxyConfig::ktls` unset. +static KTLS_UNSUPPORTED: AtomicBool = AtomicBool::new(false); + +/// Record that the kernel cannot do kTLS, so the reported mode can say why it +/// is off rather than just that it is. +pub(crate) fn mark_ktls_unsupported() { + KTLS_UNSUPPORTED.store(true, Ordering::Relaxed); +} + +/// Record the outcome of handing one connection to the kernel. +/// +/// Takes the result so both offload sites -- immediate and gated -- count the +/// same way without repeating the match. +pub(crate) fn record_ktls_offload<T, E>(result: Result<T, E>) -> Result<T, E> { + let counter = if result.is_ok() { + &KTLS_OFFLOADED + } else { + &KTLS_OFFLOAD_FAILED + }; + counter.fetch_add(1, Ordering::Relaxed); + result +} + +/// Record that one connection started splice relaying. +pub(crate) fn record_splice_engaged() { + SPLICE_ENGAGED.fetch_add(1, Ordering::Relaxed); +} + +/// Snapshot the effective acceleration state for the `Status` RPC. +pub fn accel_status(config: &ProxyConfig) -> ProxyAccelStatus { + ProxyAccelStatus { + ktls_mode: ktls_mode(config), + splice_mode: match &config.tcp_splice { + Some(splice) => splice.engage.to_string(), + None => "off".to_string(), + }, + ktls_offloaded: KTLS_OFFLOADED.load(Ordering::Relaxed), + ktls_offload_failed: KTLS_OFFLOAD_FAILED.load(Ordering::Relaxed), + splice_engaged: SPLICE_ENGAGED.load(Ordering::Relaxed), + } +} + +fn ktls_mode(config: &ProxyConfig) -> String { + describe_ktls( + config.ktls.as_ref(), + KTLS_UNSUPPORTED.load(Ordering::Relaxed), + ) +} + +/// Split out from the statics so it can be tested without mutating global state +/// that other tests in this process would then see. +fn describe_ktls(ktls: Option<&EngageAfter>, unsupported: bool) -> String { + match ktls { + Some(engage) => engage.to_string(), + None if unsupported => "disabled (kernel has no TLS ULP)".to_string(), + None => "off".to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gates_read_as_when_they_engage() { + let gate = |bytes, secs: Option<u64>| EngageAfter { + after_bytes: bytes, + after_duration: secs.map(std::time::Duration::from_secs), + }; + assert_eq!(gate(None, None).to_string(), "immediate"); + assert_eq!(gate(Some(65536), None).to_string(), "after 64 KiB"); + assert_eq!(gate(Some(1 << 20), None).to_string(), "after 1 MiB"); + assert_eq!(gate(Some(1000), None).to_string(), "after 1000 B"); + assert_eq!(gate(None, Some(5)).to_string(), "after 5s"); + assert_eq!(gate(Some(65536), Some(5)).to_string(), "after 64 KiB or 5s"); + } + + #[test] + fn a_probe_disabled_gateway_does_not_look_unconfigured() { + assert_eq!(describe_ktls(None, false), "off"); + assert_eq!( + describe_ktls(None, true), + "disabled (kernel has no TLS ULP)" + ); + // A kernel that cannot do kTLS is only interesting while kTLS is off; + // once it is on, the gate is the useful thing to report. + let gate = EngageAfter::default(); + assert_eq!(describe_ktls(Some(&gate), true), "immediate"); + } +} diff --git a/dstack/gateway/src/proxy/tls_passthough.rs b/dstack/gateway/src/proxy/tls_passthough.rs index 3848883d1..21311730c 100644 --- a/dstack/gateway/src/proxy/tls_passthough.rs +++ b/dstack/gateway/src/proxy/tls_passthough.rs @@ -3,11 +3,14 @@ // SPDX-License-Identifier: Apache-2.0 use std::fmt::Debug; +use std::net::SocketAddr; use std::sync::atomic::Ordering; use std::time::Duration; use anyhow::{bail, Context, Result}; +use hickory_resolver::config::{NameServerConfig, ResolverConfig}; use hickory_resolver::lookup::Lookup; +use hickory_resolver::net::runtime::TokioRuntimeProvider; use hickory_resolver::proto::rr::RData; use hickory_resolver::TokioResolver; use proxy_protocol::ProxyHeader; @@ -20,7 +23,7 @@ use crate::{ }; use super::{ - io_bridge::bridge, + io_bridge::bridge_tcp, port_policy::{filter_allowed_addresses, should_send_pp}, AddressGroup, }; @@ -59,11 +62,11 @@ pub(crate) struct AppAddressResolver { } impl AppAddressResolver { - pub(crate) fn new(prefix: String, compat: bool) -> Result<Self> { + pub(crate) fn new(prefix: String, compat: bool, dns_servers: Vec<SocketAddr>) -> Result<Self> { Ok(Self { prefix, compat, - resolver: app_address_tokio_resolver_from_system_conf()?, + resolver: app_address_tokio_resolver(dns_servers)?, }) } @@ -72,8 +75,25 @@ impl AppAddressResolver { } } -fn app_address_tokio_resolver_from_system_conf() -> Result<TokioResolver> { - let mut builder = TokioResolver::builder_tokio().context("failed to read system dns config")?; +fn app_address_tokio_resolver(dns_servers: Vec<SocketAddr>) -> Result<TokioResolver> { + let mut builder = if dns_servers.is_empty() { + TokioResolver::builder_tokio().context("failed to read system dns config")? + } else { + let name_servers = dns_servers + .into_iter() + .map(|dns_server| { + let mut name_server = NameServerConfig::udp_and_tcp(dns_server.ip()); + for connection in &mut name_server.connections { + connection.port = dns_server.port(); + } + name_server + }) + .collect(); + TokioResolver::builder_with_config( + ResolverConfig::from_parts(None, Vec::new(), name_servers), + TokioRuntimeProvider::default(), + ) + }; // App-address records may appear shortly after a CVM/app is registered. // Reusing one resolver enables positive TXT caching, but we do not want a @@ -199,8 +219,29 @@ pub(crate) async fn connect_multiple_hosts( ) -> Result<(TcpStream, EnteredCounter, String)> { check_connection_limit(&addresses, max_connections, app_id)?; + let mut candidates = addresses.into_iter(); + let Some(first) = candidates.next() else { + bail!("no addresses to connect to app <{app_id}>"); + }; + + // Fast path: with a single candidate there is nothing to race, so skip the + // JoinSet and the task spawn it needs. That allocation and scheduling + // happened on every connection, and single-address apps are the common + // case. + if candidates.as_slice().is_empty() { + let addr = first; + let counter = addr.counter.enter(); + let ip = addr.ip; + debug!("connecting to {ip}:{port}"); + let connection = TcpStream::connect((ip, port)) + .await + .map_err(|e| anyhow::anyhow!("failed to connect to app@{ip}:{port}: {e}"))?; + let _ = connection.set_nodelay(true); + return Ok((connection, counter, addr.instance_id)); + } + let mut join_set = JoinSet::new(); - for addr in addresses { + for addr in std::iter::once(first).chain(candidates) { let counter = addr.counter.enter(); let ip = addr.ip; let instance_id = addr.instance_id; @@ -228,6 +269,9 @@ pub(crate) async fn connect_multiple_hosts( } } }; + // Disable Nagle on the upstream socket for the same reason as the inbound + // side: avoid delayed-ACK stalls on small proxied messages. + let _ = connection.set_nodelay(true); debug!("connected to {:?}", connection.peer_addr()); Ok((connection, counter, instance_id)) } @@ -259,9 +303,31 @@ pub(crate) async fn proxy_to_app( .write_all(&buffer) .await .context("failed to write to app")?; - bridge(inbound, outbound, &state.config.proxy) - .await - .context("failed to copy between inbound and outbound")?; + if let Some(gate) = &state.config.proxy.tcp_splice { + // Passthrough is a pure TCP relay: move bytes kernel-side with splice. + // Both ends are plain sockets here, so a FIN is the whole close. + let idle = state.config.proxy.idle_timeout(); + if gate.engage.is_immediate() { + super::splice::splice_bidirectional( + inbound, + outbound, + gate.release_idle_pipes, + idle, + super::splice::CloseKind::Tcp, + ) + .await + .context("failed to splice between inbound and outbound")?; + } else { + let buf_size = state.config.proxy.buffer_size; + super::splice::splice_bidirectional_after(inbound, outbound, gate, buf_size, idle) + .await + .context("failed to relay between inbound and outbound")?; + } + } else { + bridge_tcp(inbound, outbound, &state.config.proxy) + .await + .context("failed to copy between inbound and outbound")?; + } Ok(()) } @@ -271,7 +337,7 @@ mod tests { #[tokio::test] async fn test_resolve_app_address() -> Result<()> { - let resolver = AppAddressResolver::new("_dstack-app-address".to_string(), false)?; + let resolver = AppAddressResolver::new("_dstack-app-address".to_string(), false, vec![])?; let app_addr = resolver .resolve("3327603e03f5bd1f830812ca4a789277fc31f577.app.dstack.org") .await?; diff --git a/dstack/gateway/src/proxy/tls_terminate.rs b/dstack/gateway/src/proxy/tls_terminate.rs index 48136b1db..430089977 100644 --- a/dstack/gateway/src/proxy/tls_terminate.rs +++ b/dstack/gateway/src/proxy/tls_terminate.rs @@ -16,7 +16,7 @@ use hyper_util::rt::tokio::TokioIo; use proxy_protocol::ProxyHeader; use rustls::version::{TLS12, TLS13}; use serde::Serialize; -use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt as _, ReadBuf}; +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf}; use tokio::net::TcpStream; use tokio::time::timeout; use tokio_rustls::{rustls, server::TlsStream, TlsAcceptor}; @@ -110,6 +110,15 @@ pub(crate) fn create_acceptor_with_cert_resolver( CryptoProvider::AwsLcRs => rustls::crypto::aws_lc_rs::default_provider(), CryptoProvider::Ring => rustls::crypto::ring::default_provider(), }; + // Stateless session tickets. TLS 1.2 resumption already works via the + // in-memory session-ID cache, but TLS 1.3 resumption requires a ticketer; + // without one, every reconnect pays a full handshake (a large RSA signing + // cost on the server). Installing a ticketer restores resumption for 1.3. + let ticketer = match proxy_config.tls_crypto_provider { + CryptoProvider::AwsLcRs => rustls::crypto::aws_lc_rs::Ticketer::new(), + CryptoProvider::Ring => rustls::crypto::ring::Ticketer::new(), + } + .context("failed to create TLS session ticketer")?; let supported_versions = proxy_config .tls_versions .iter() @@ -125,6 +134,15 @@ pub(crate) fn create_acceptor_with_cert_resolver( .with_no_client_auth() .with_cert_resolver(cert_resolver); + config.ticketer = ticketer; + + // kTLS needs the negotiated traffic secrets so it can install them into + // the kernel's TLS ULP. This is opt-in because it moves session keys + // outside rustls' control (see the `ktls` config docs). + if proxy_config.ktls.is_some() { + config.enable_secret_extraction = true; + } + if h2 { config.alpn_protocols = vec![b"h2".to_vec()]; } @@ -270,6 +288,66 @@ impl Proxy { Ok(tls_stream) } + /// Accept a TLS connection keeping the `CorkStream` wrapper, so the + /// connection can be handed to the kernel later without re-wrapping. + async fn tls_accept_corked( + &self, + inbound: TcpStream, + buffer: Vec<u8>, + h2: bool, + ) -> Result<TlsStream<ktls::CorkStream<MergedStream>>> { + let stream = ktls::CorkStream::new(MergedStream { + buffer, + buffer_cursor: 0, + inbound, + }); + let acceptor = if h2 { + &self.h2_acceptor + } else { + &self.acceptor + }; + timeout( + self.config.proxy.timeouts.handshake, + acceptor.accept(stream), + ) + .await + .context("handshake timeout")? + .context("failed to accept tls connection") + } + + /// Accept a TLS connection and hand the socket over to kernel TLS. + /// + /// The handshake still runs in rustls; afterwards the negotiated keys are + /// installed into the kernel TLS ULP so record encryption happens there. + /// The inner IO must be wrapped in `CorkStream` because that is the only + /// way to drain a rustls stream cleanly at a record boundary. + async fn tls_accept_ktls( + &self, + inbound: TcpStream, + buffer: Vec<u8>, + h2: bool, + ) -> Result<ktls::KtlsStream<MergedStream>> { + let stream = ktls::CorkStream::new(MergedStream { + buffer, + buffer_cursor: 0, + inbound, + }); + let acceptor = if h2 { + &self.h2_acceptor + } else { + &self.acceptor + }; + let tls_stream = timeout( + self.config.proxy.timeouts.handshake, + acceptor.accept(stream), + ) + .await + .context("handshake timeout")? + .context("failed to accept tls connection")?; + super::stats::record_ktls_offload(ktls::config_ktls_server(tls_stream).await) + .context("failed to enable kernel TLS") + } + pub(super) async fn proxy( &self, inbound: TcpStream, @@ -291,20 +369,136 @@ impl Proxy { .with_context(|| format!("app <{app_id}> not found"))?; let addresses = filter_allowed_addresses(self, addresses, app_id, port)?; debug!("selected top n hosts: {addresses:?}"); - let tls_stream = self.tls_accept(inbound, buffer, h2).await?; + if let Some(ktls) = &self.config.proxy.ktls { + let splice = self.config.proxy.tcp_splice.as_ref(); + // A gated offload only pays off if the socket is spliced afterwards, + // so it needs both sections configured. + if let Some(splice) = splice.filter(|_| !ktls.is_immediate()) { + // Adaptive: stay in userspace rustls until the connection proves + // itself worth the offload, then hand it to the kernel. + let tls_stream = self.tls_accept_corked(inbound, buffer, h2).await?; + let (mut outbound, _counter, instance_id) = + self.connect_upstream(addresses, port, app_id).await?; + self.send_pp_header(&mut outbound, &instance_id, port, pp_header) + .await?; + return super::adaptive_ktls::relay_with_adaptive_offload( + tls_stream, + outbound, + ktls, + splice, + self.config.proxy.idle_timeout(), + ) + .await; + } + let tls_stream = self.tls_accept_ktls(inbound, buffer, h2).await?; + if let Some(splice) = splice { + // With kTLS the socket carries plaintext from userspace's point + // of view, so the payload can be relayed with splice and never + // enters this process at all. + let (mut outbound, _counter, instance_id) = + self.connect_upstream(addresses, port, app_id).await?; + self.send_pp_header(&mut outbound, &instance_id, port, pp_header) + .await?; + let (drained, stream) = tls_stream.into_raw(); + let (buffered, tcp) = stream.into_parts(); + // These two are not the same kind of data and must not be + // treated as interchangeable: `drained` is plaintext rustls + // decrypted past the handshake and owes to the app, while + // `buffered` is whatever raw *ciphertext* was left in the SNI + // sniff buffer. Forwarding the latter would hand the app TLS + // records to interpret as application data. + // + // A completed handshake always consumes the sniff buffer, so + // this is unreachable rather than merely unlikely -- but it is + // cheap to refuse instead of finding out by corrupting a stream. + if !buffered.is_empty() { + bail!( + "{} bytes of unconsumed ciphertext at kTLS handover", + buffered.len() + ); + } + // Plaintext rustls already decrypted has to reach the app before + // the kernel starts moving bytes directly. + let drained = drained.unwrap_or_default(); + if !drained.is_empty() { + outbound + .write_all(&drained) + .await + .context("failed to flush drained data to app")?; + } + // `tcp` is now a kernel-TLS socket, so closing it needs a + // close_notify and not just a FIN. + return super::splice::splice_bidirectional( + tcp, + outbound, + splice.release_idle_pipes, + self.config.proxy.idle_timeout(), + super::splice::CloseKind::KernelTls, + ) + .await + .context("ktls splice error"); + } + self.relay_to_app(tls_stream, addresses, port, app_id, pp_header) + .await + } else { + let tls_stream = self.tls_accept(inbound, buffer, h2).await?; + self.relay_to_app(tls_stream, addresses, port, app_id, pp_header) + .await + } + } + + /// Connect to the app and relay an already-terminated TLS stream to it. + /// + /// Generic over the accepted stream so the userspace-rustls and kTLS + /// paths share the same connect / PROXY-protocol / bridging logic. + /// Race a connection to the app's top-N addresses. + async fn connect_upstream( + &self, + addresses: super::AddressGroup, + port: u16, + app_id: &str, + ) -> Result<(TcpStream, crate::models::EnteredCounter, String)> { let max_connections = self.config.proxy.max_connections_per_app; - let (mut outbound, _counter, instance_id) = timeout( + timeout( self.config.proxy.timeouts.connect, connect_multiple_hosts(addresses, port, max_connections, app_id), ) .await .map_err(|_| anyhow!("connecting timeout"))? - .context("failed to connect to app")?; - if should_send_pp(self, &instance_id, port) { + .context("failed to connect to app") + } + + /// Forward the client's address to the app when its port policy asks for it. + async fn send_pp_header( + &self, + outbound: &mut TcpStream, + instance_id: &str, + port: u16, + pp_header: ProxyHeader, + ) -> Result<()> { + if should_send_pp(self, instance_id, port) { let pp_header_bin = proxy_protocol::encode(pp_header).context("failed to encode pp header")?; outbound.write_all(&pp_header_bin).await?; } + Ok(()) + } + + async fn relay_to_app<S>( + &self, + tls_stream: S, + addresses: super::AddressGroup, + port: u16, + app_id: &str, + pp_header: ProxyHeader, + ) -> Result<()> + where + S: AsyncRead + AsyncWrite + Unpin, + { + let (mut outbound, _counter, instance_id) = + self.connect_upstream(addresses, port, app_id).await?; + self.send_pp_header(&mut outbound, &instance_id, port, pp_header) + .await?; bridge( IgnoreUnexpectedEofStream::new(tls_stream), outbound, @@ -316,6 +510,17 @@ impl Proxy { } } +/// Give up the raw socket *and* anything still buffered in front of it. +/// +/// Deliberately not `Into<TcpStream>`: that conversion existed, and it dropped +/// the remainder silently. Whatever is left here is raw ciphertext from the SNI +/// sniff, which cannot be forwarded to an app expecting plaintext and cannot be +/// pushed back once the socket belongs to the kernel -- so the only safe thing +/// is to make every caller look at it. +pub(crate) trait SocketParts { + fn into_socket_parts(self) -> (Vec<u8>, TcpStream); +} + #[pin_project::pin_project] struct MergedStream { buffer: Vec<u8>, @@ -346,6 +551,37 @@ impl AsyncRead for MergedStream { this.inbound.poll_read(cx, buf) } } +impl MergedStream { + /// Unwrap to the raw socket, returning any bytes still buffered from the + /// pre-handshake sniff. After a completed handshake the buffer is drained, + /// so the returned `Vec` is normally empty; callers that bypass the + /// `AsyncRead` impl (e.g. splice) must still handle a non-empty remainder. + fn into_parts(self) -> (Vec<u8>, TcpStream) { + let remaining = self.buffer[self.buffer_cursor.min(self.buffer.len())..].to_vec(); + (remaining, self.inbound) + } +} + +impl SocketParts for MergedStream { + fn into_socket_parts(self) -> (Vec<u8>, TcpStream) { + self.into_parts() + } +} + +impl std::os::fd::AsRawFd for MergedStream { + fn as_raw_fd(&self) -> std::os::fd::RawFd { + self.inbound.as_raw_fd() + } +} + +impl ktls::AsyncReadReady for MergedStream { + fn poll_read_ready(&self, cx: &mut Context<'_>) -> Poll<io::Result<()>> { + // Safe to defer to the socket: by the time kTLS takes over, the + // buffered ClientHello prefix has already been consumed by rustls. + self.inbound.poll_read_ready(cx) + } +} + impl AsyncWrite for MergedStream { fn poll_write( self: std::pin::Pin<&mut Self>, @@ -381,3 +617,46 @@ impl AsyncWrite for MergedStream { self.inbound.is_write_vectored() } } + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::AsyncReadExt as _; + use tokio::net::TcpListener; + + async fn merged_with(buffer: Vec<u8>) -> MergedStream { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("addr"); + let client = TcpStream::connect(addr).await.expect("connect"); + drop(client); + let (inbound, _) = listener.accept().await.expect("accept"); + MergedStream { + buffer, + buffer_cursor: 0, + inbound, + } + } + + /// The kTLS handover reads the remainder through this, and forwarding raw + /// ciphertext to an app expecting plaintext is the failure it guards + /// against -- so an unconsumed sniff buffer has to be visible, not silently + /// swallowed by the unwrap. + #[tokio::test] + async fn an_unconsumed_sniff_buffer_is_surfaced_not_dropped() { + let stream = merged_with(b"leftover ciphertext".to_vec()).await; + let (remainder, _socket) = stream.into_socket_parts(); + assert_eq!(remainder, b"leftover ciphertext"); + } + + /// The normal case: rustls drains the sniff buffer during the handshake, so + /// the handover sees nothing left and proceeds. + #[tokio::test] + async fn a_consumed_sniff_buffer_leaves_no_remainder() { + let mut stream = merged_with(b"clienthello".to_vec()).await; + let mut sink = vec![0u8; 11]; + stream.read_exact(&mut sink).await.expect("read"); + assert_eq!(&sink, b"clienthello"); + let (remainder, _socket) = stream.into_socket_parts(); + assert!(remainder.is_empty(), "got {remainder:?}"); + } +} diff --git a/dstack/gateway/src/time.rs b/dstack/gateway/src/time.rs new file mode 100644 index 000000000..71e29a376 --- /dev/null +++ b/dstack/gateway/src/time.rs @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Epoch-seconds conversions shared across the gateway. +//! +//! Timestamps cross the KV boundary as `u64` seconds since the Unix epoch and +//! are held in memory as `SystemTime`, so both directions are needed in several +//! modules. Keeping one pair of them means the saturating behaviour — a time +//! before the epoch reads as 0, a count of seconds `SystemTime` cannot +//! represent clamps to the epoch — is decided once instead of at each call +//! site. +//! +//! Call sites that would rather fail than saturate keep their own +//! `duration_since(UNIX_EPOCH)?`: a clock behind the epoch is a real fault, and +//! whether to report it or carry on is the caller's decision, not this +//! module's. + +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// Seconds since the Unix epoch for `ts`, or 0 if `ts` predates the epoch. +pub(crate) fn encode_ts(ts: SystemTime) -> u64 { + ts.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs() +} + +/// The instant `ts` seconds after the Unix epoch, clamped to the epoch when +/// that instant is not representable. +pub(crate) fn decode_ts(ts: u64) -> SystemTime { + UNIX_EPOCH + .checked_add(Duration::from_secs(ts)) + .unwrap_or(UNIX_EPOCH) +} + +/// The local wall clock, as seconds since the Unix epoch. +pub(crate) fn now_secs() -> u64 { + encode_ts(SystemTime::now()) +} diff --git a/dstack/gateway/src/web_routes.rs b/dstack/gateway/src/web_routes.rs index 5f72735db..29b37d9a9 100644 --- a/dstack/gateway/src/web_routes.rs +++ b/dstack/gateway/src/web_routes.rs @@ -4,8 +4,9 @@ use crate::main_service::Proxy; use anyhow::Result; -use rocket::{get, response::content::RawHtml, routes, Route, State}; +use rocket::{get, http::ContentType, response::content::RawHtml, routes, Route, State}; +mod metrics; mod route_index; mod wavekv_sync; @@ -14,13 +15,26 @@ async fn index(state: &State<Proxy>) -> Result<RawHtml<String>, String> { route_index::index(state).await.map_err(|e| format!("{e}")) } +/// Prometheus scrape endpoint. +/// +/// Mounted on the admin listener only: the series below name domains, node ids +/// and instance counts, which is topology no unauthenticated caller should be +/// able to read. +#[get("/metrics")] +fn scrape(state: &State<Proxy>) -> (ContentType, String) { + // Naming the exposition version lets a scraper pick its parser instead of + // inferring one from a bare `text/plain`. + let content_type = ContentType::new("text", "plain").with_params([("version", "0.0.4")]); + (content_type, metrics::render(state)) +} + #[get("/health")] fn health() -> &'static str { "OK" } pub fn routes() -> Vec<Route> { - routes![index] + routes![index, scrape] } /// Health endpoint for simple liveness checks @@ -30,5 +44,40 @@ pub fn health_routes() -> Vec<Route> { /// WaveKV sync endpoint (for main server, requires mTLS gateway auth) pub fn wavekv_sync_routes() -> Vec<Route> { - routes![wavekv_sync::sync_store] + routes![wavekv_sync::sync_store, wavekv_sync::push_store] +} +#[cfg(test)] +mod tests { + use super::*; + + /// The scrape output names domains, node ids and instance counts, so `/metrics` + /// must be part of the authenticated admin route set (`routes()`) and must not + /// be included in the route sets intended for non-admin listeners. + /// + /// This test checks the route-set membership only; it does not inspect Rocket + /// listener wiring in `main.rs`. + #[test] + fn metrics_is_mounted_on_the_admin_listener_only() { + let mounted = |set: Vec<Route>| set.iter().any(|route| route.uri.path() == "/metrics"); + assert!(mounted(routes())); + assert!(!mounted(health_routes())); + assert!(!mounted(wavekv_sync_routes())); + } + + /// A typo in either path would prevent peers from synchronizing. + #[test] + fn the_sync_routes_are_mounted_where_peers_look_for_them() { + let mounted: Vec<String> = wavekv_sync_routes() + .iter() + .map(|route| route.uri.to_string()) + .collect(); + + for expected in ["/wavekv/sync/<store>", "/wavekv/push/<store>"] { + assert!( + mounted.iter().any(|uri| uri == expected), + "{expected} is not mounted; peers would be unable to synchronize. \ + mounted: {mounted:?}" + ); + } + } } diff --git a/dstack/gateway/src/web_routes/metrics.rs b/dstack/gateway/src/web_routes/metrics.rs new file mode 100644 index 000000000..bdad9cbcb --- /dev/null +++ b/dstack/gateway/src/web_routes/metrics.rs @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! `/metrics` scrape handler. + +use std::sync::atomic::Ordering; + +use rocket::State; + +use crate::{ + main_service::Proxy, + metrics::{self, PeerSnapshot, Snapshot, StoreSnapshot}, + proxy::{stats::accel_status, NUM_CONNECTIONS}, +}; + +pub fn render(state: &State<Proxy>) -> String { + metrics::render(&sample(state)) +} + +fn sample(state: &State<Proxy>) -> Snapshot { + // Everything below reads replicated records through the decoding helpers + // that feed `record_decode_failure`. A scrape reporting a counter must not + // also be a writer of it. + let _sampling = metrics::scrape_guard(); + + let kv_store = state.kv_store().clone(); + let accel = accel_status(&state.config.proxy); + + // The public data path takes this lock on every connection, so the scrape + // holds it for one O(1) count and nothing else. + // + // The node counts deliberately do not go through `get_all_nodes()` / + // `get_active_nodes()`: those read no proxy state at all -- only + // `self.kv_store` -- so routing them through the lock would drag two loads + // of the node table, a `GatewayNodeInfo` per node, and one ephemeral-lock + // acquisition per node for an unused `last_seen` in here with them. + let instances = state.lock().state.instances.len() as u64; + let (nodes_total, nodes_active) = kv_store.count_nodes(); + + let stores = vec![ + store_snapshot("persistent", kv_store.persistent()), + store_snapshot("ephemeral", kv_store.ephemeral()), + ]; + + let cert_not_after = kv_store + .load_all_cert_data() + .into_iter() + .map(|(domain, data)| (domain, data.not_after)) + .collect(); + + // One pass over the node table plus one ephemeral read per node -- the + // cost the counting fast-path above avoids, paid here deliberately, + // because a per-node timestamp is the point: a node whose value stops + // advancing is offline, however the operator's alerting words it. + let node_last_seen = kv_store + .load_all_nodes() + .into_keys() + .filter_map(|id| { + kv_store + .get_node_latest_last_seen(id) + .map(|last_seen| (id, last_seen)) + }) + .collect(); + + Snapshot { + version: crate::app_version(), + node_id: kv_store.my_node_id(), + instances, + connections: NUM_CONNECTIONS.load(Ordering::Relaxed), + nodes_total, + nodes_active, + accel, + stores, + cert_not_after, + node_last_seen, + } +} + +fn store_snapshot(name: &'static str, node: &wavekv::node::Node) -> StoreSnapshot { + // `status()` is O(peers) on wavekv 1.x, so this is cheap enough to do under + // the read lock. On the 2.0 branch it also computes a state digest over the + // whole dataset -- at that point this call hashes the entire store, twice + // per scrape, while blocking writers. Revisit when the dependency moves. + let status = node.read().status(); + StoreSnapshot { + name, + keys: status.n_kvs as u64, + next_seq: status.next_seq, + dirty: status.dirty, + peers: status + .peers + .into_iter() + .map(|peer| PeerSnapshot { + id: peer.id, + local_ack: peer.ack, + peer_ack: peer.peer_ack, + }) + .collect(), + } +} diff --git a/dstack/gateway/src/web_routes/route_index.rs b/dstack/gateway/src/web_routes/route_index.rs index db414391f..c4e9907e0 100644 --- a/dstack/gateway/src/web_routes/route_index.rs +++ b/dstack/gateway/src/web_routes/route_index.rs @@ -23,7 +23,12 @@ pub async fn index(state: &State<Proxy>) -> anyhow::Result<Html<String>> { .acme_info() .await .context("Failed to get ACME info")?; - let model = Dashboard { status, acme_info }; + let accel = status.accel.clone().unwrap_or_default(); + let model = Dashboard { + status, + acme_info, + accel, + }; let html = model.render().context("Failed to render template")?; Ok(Html(html)) } diff --git a/dstack/gateway/src/web_routes/wavekv_sync.rs b/dstack/gateway/src/web_routes/wavekv_sync.rs index dead1141c..362748da3 100644 --- a/dstack/gateway/src/web_routes/wavekv_sync.rs +++ b/dstack/gateway/src/web_routes/wavekv_sync.rs @@ -7,60 +7,41 @@ //! Sync data is encoded using msgpack + gzip compression for efficiency. use crate::{ - kv::{decode, encode}, + kv::{gunzip_bounded, MAX_COMPRESSED_SYNC_BYTES, MAX_DECOMPRESSED_SYNC_BYTES}, main_service::Proxy, }; -use flate2::{read::GzDecoder, write::GzEncoder, Compression}; +use flate2::{write::GzEncoder, Compression}; use ra_tls::traits::CertExt; use rocket::{ data::{Data, ToByteUnit}, http::{ContentType, Status}, - mtls::{oid::Oid, Certificate}, + mtls::{oid::Oid, x509::X509Extension, Certificate}, post, State, }; -use std::io::{Read, Write}; +use std::io::Write; use tracing::warn; -use wavekv::sync::{SyncMessage, SyncResponse}; +use wavekv::sync::SyncEnvelope; -/// Wrapper to implement CertExt for Rocket's Certificate -struct RocketCert<'a>(&'a Certificate<'a>); +/// Adapter implementing `CertExt` over a parsed certificate's extension list. +/// +/// It holds the extensions rather than the `Certificate` so that a test can build one: +/// `rocket::mtls::Certificate` has no public constructor — it can only be produced by a +/// real mTLS handshake — while an extension list comes straight out of `X509Certificate`. +struct RocketCert<'a, 'b>(&'b [X509Extension<'a>]); -impl CertExt for RocketCert<'_> { +impl CertExt for RocketCert<'_, '_> { fn get_extension_der(&self, oid: &[u64]) -> anyhow::Result<Option<Vec<u8>>> { let oid = Oid::from(oid).map_err(|_| anyhow::anyhow!("failed to create OID from slice"))?; - let Some(ext) = self.0.extensions().iter().find(|ext| ext.oid == oid) else { + let Some(ext) = self.0.iter().find(|ext| ext.oid == oid) else { return Ok(None); }; Ok(Some(ext.value.to_vec())) } } -/// Decode compressed msgpack data -fn decode_sync_message(data: &[u8]) -> Result<SyncMessage, Status> { - // Decompress - let mut decoder = GzDecoder::new(data); - let mut decompressed = Vec::new(); - decoder.read_to_end(&mut decompressed).map_err(|e| { - warn!("failed to decompress sync message: {e}"); - Status::BadRequest - })?; - - decode(&decompressed).map_err(|e| { - warn!("failed to decode sync message: {e}"); - Status::BadRequest - }) -} - -/// Encode and compress sync response -fn encode_sync_response(response: &SyncResponse) -> Result<Vec<u8>, Status> { - let encoded = encode(response).map_err(|e| { - warn!("failed to encode sync response: {e}"); - Status::InternalServerError - })?; - - // Compress +fn gzip(bytes: &[u8]) -> Result<Vec<u8>, Status> { let mut encoder = GzEncoder::new(Vec::new(), Compression::fast()); - encoder.write_all(&encoded).map_err(|e| { + encoder.write_all(bytes).map_err(|e| { warn!("failed to compress sync response: {e}"); Status::InternalServerError })?; @@ -70,41 +51,90 @@ fn encode_sync_response(response: &SyncResponse) -> Result<Vec<u8>, Status> { }) } -/// Verify that the request is from a gateway with the same app_id (mTLS verification) -fn verify_gateway_peer(state: &Proxy, cert: Option<Certificate<'_>>) -> Result<(), Status> { - // Skip verification if not running in dstack (test mode) - if state.config.debug.insecure_skip_attestation { - return Ok(()); +fn gunzip(data: &[u8]) -> Result<Vec<u8>, Status> { + gunzip_bounded(data, MAX_DECOMPRESSED_SYNC_BYTES).map_err(|e| { + warn!("failed to decompress sync payload: {e:#}"); + Status::BadRequest + }) +} + +async fn read_compressed_body(data: Data<'_>) -> Result<Vec<u8>, Status> { + let bytes = data + .open(MAX_COMPRESSED_SYNC_BYTES.bytes()) + .into_bytes() + .await + .map_err(|_| Status::BadRequest)?; + if !bytes.is_complete() { + warn!("sync payload exceeds the {MAX_COMPRESSED_SYNC_BYTES}-byte compressed-size limit"); + return Err(Status::PayloadTooLarge); } + Ok(bytes.into_inner()) +} + +/// Read a sync envelope from a bounded request body. +/// Decode a gzipped sync envelope. +/// +/// Split from the `Data` read so the request-handling half of these endpoints +/// can be driven from a test. Rocket's local client presents no certificate, so +/// a test that went through the real route would be stopped at the peer check +/// before reaching any of this. +fn decode_envelope(compressed: &[u8]) -> Result<SyncEnvelope, Status> { + let decompressed = gunzip(compressed)?; + // `SyncEnvelope::decode` enforces the schema version and rejects trailing bytes; + // it is deliberately not the generic `decode` used for KV values. + SyncEnvelope::decode(&decompressed).map_err(|e| { + warn!("failed to decode sync envelope: {e:#}"); + Status::BadRequest + }) +} +/// Verify that the request is from a gateway with the same app_id (mTLS verification) +fn verify_gateway_peer(state: &Proxy, cert: Option<Certificate<'_>>) -> Result<(), Status> { let Some(cert) = cert else { warn!("WaveKV sync: client certificate required but not provided"); return Err(Status::Unauthorized); }; - let remote_app_id = RocketCert(&cert).get_app_id().map_err(|e| { + authorize_peer(&RocketCert(cert.extensions()), state.my_app_id()) +} + +/// Decide whether a certificate's app identity is one we accept. +/// +/// Split out from `verify_gateway_peer` because that function's other half — Rocket's +/// certificate guard — cannot be exercised from a test, which left this decision, the +/// actual authorization rule, uncovered. +/// +/// The app-id extension is the only identity accepted. There used to be a fallback to +/// the app-info extension here, and it existed because certificates issued through a +/// local CA carried app-info but not app-id — so without it, clustering under any +/// non-KMS key provider failed. That is fixed at the source now (`local_ca_app_id` in +/// cert-client stamps the app id the way KMS does), which matters because the fallback +/// only ever covered *this* side of the connection: `AppIdValidator` in the sync client +/// and `ensure_from_gateway` in the RPC handler both read app-id with no fallback, so a +/// certificate the fallback rescued here was still rejected in both other places. One +/// rule across all three is worth more than a second path that was untested, disagreed +/// with its neighbours, and read an issuer claim the cluster does not otherwise +/// authorize on. +fn authorize_peer(cert: &impl CertExt, my_app_id: &[u8]) -> Result<(), Status> { + let remote_app_id = cert.get_app_id().map_err(|e| { warn!("WaveKV sync: failed to extract app_id from certificate: {e}"); Status::Unauthorized })?; let Some(remote_app_id) = remote_app_id else { - warn!("WaveKV sync: certificate does not contain app_id"); + warn!("WaveKV sync: certificate does not contain an app_id extension"); return Err(Status::Unauthorized); }; - if state.my_app_id() != Some(remote_app_id.as_slice()) { - warn!( - "WaveKV sync: app_id mismatch, expected {:?}, got {:?}", - state.my_app_id(), - remote_app_id - ); + if my_app_id != remote_app_id.as_slice() { + warn!("WaveKV sync: app_id mismatch, expected {my_app_id:?}, got {remote_app_id:?}"); return Err(Status::Forbidden); } Ok(()) } -/// Handle sync request (msgpack + gzip encoded) +/// WaveKV sync endpoint. #[post("/wavekv/sync/<store>", data = "<data>")] pub async fn sync_store( state: &State<Proxy>, @@ -113,38 +143,738 @@ pub async fn sync_store( data: Data<'_>, ) -> Result<(ContentType, Vec<u8>), Status> { verify_gateway_peer(state, cert)?; + let body = read_compressed_body(data).await?; + handle_sync(state, store, &body) +} +/// Everything `sync_store` does once the caller is known to be a peer. +pub(crate) fn handle_sync( + state: &Proxy, + store: &str, + body: &[u8], +) -> Result<(ContentType, Vec<u8>), Status> { let Some(ref wavekv_sync) = state.wavekv_sync else { return Err(Status::ServiceUnavailable); }; - // Read and decode request - let bytes = data - .open(16.mebibytes()) - .into_bytes() - .await - .map_err(|_| Status::BadRequest)?; - let msg = decode_sync_message(&bytes)?; - - // Reject sync from node_id == 0 - if msg.sender_id == 0 { + let env = decode_envelope(body)?; + if env.sender_id == 0 { warn!("rejected sync from invalid node_id 0"); return Err(Status::BadRequest); } + refuse_removed_sender(state, env.sender_id)?; + + let Some(result) = wavekv_sync.handle_envelope(store, env) else { + return Err(Status::NotFound); + }; + let response = result.map_err(|e| { + tracing::error!("{store} sync failed: {e:#}"); + Status::InternalServerError + })?; + + let encoded = response.encode().map_err(|e| { + warn!("failed to encode sync envelope: {e:#}"); + Status::InternalServerError + })?; + Ok(( + ContentType::new("application", "x-msgpack-gz"), + gzip(&encoded)?, + )) +} - // Handle sync based on store type - let response = match store { - "persistent" => wavekv_sync.handle_persistent_sync(msg), - "ephemeral" => wavekv_sync.handle_ephemeral_sync(msg), - _ => return Err(Status::NotFound), +/// Refuse an envelope from a node an operator has removed. +/// +/// The app-identity check above proves the sender is *a* gateway of this app; +/// it cannot prove the sender is still a *member*. A removed node returning +/// with its old data directory diverges from every digest, and wavekv's +/// repair would answer with a full re-exchange that resurrects every record +/// whose delete the cluster has already collected -- so the door, not the +/// merge, is where a removed sender has to stop. Re-admission is an explicit +/// operator decision: SetNodeUrl clears the marker. +fn refuse_removed_sender(state: &Proxy, sender_id: u32) -> Result<(), Status> { + if state.kv_store().is_peer_removed(sender_id) { + warn!("refused an envelope from removed node {sender_id}"); + return Err(Status::Forbidden); } - .map_err(|e| { - tracing::error!("{store} sync failed: {e}"); + Ok(()) +} + +/// Opportunistic push endpoint (wavekv RFC 0001 section 3.9). +/// +/// Entries only: the receiver merges data but never moves its ack coverage from this +/// channel, so loss, duplication and reordering here are all harmless and the periodic +/// round remains the anti-entropy backstop. +#[post("/wavekv/push/<store>", data = "<data>")] +pub async fn push_store( + state: &State<Proxy>, + cert: Option<Certificate<'_>>, + store: &str, + data: Data<'_>, +) -> Result<Status, Status> { + verify_gateway_peer(state, cert)?; + let body = read_compressed_body(data).await?; + handle_push(state, store, &body) +} + +/// Everything `push_store` does once the caller is known to be a peer. +pub(crate) fn handle_push(state: &Proxy, store: &str, body: &[u8]) -> Result<Status, Status> { + let Some(ref wavekv_sync) = state.wavekv_sync else { + return Err(Status::ServiceUnavailable); + }; + + let env = decode_envelope(body)?; + if env.sender_id == 0 { + warn!("rejected push from invalid node_id 0"); + return Err(Status::BadRequest); + } + refuse_removed_sender(state, env.sender_id)?; + + let Some(result) = wavekv_sync.handle_push(store, env) else { + return Err(Status::NotFound); + }; + result.map_err(|e| { + tracing::error!("{store} push failed: {e:#}"); Status::InternalServerError })?; + Ok(Status::Ok) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{load_config_figment, Config, MutualConfig, TlsConfig}; + use crate::kv::{HttpsClient, NodeData}; + use crate::main_service::{Proxy, ProxyOptions}; + use rocket::local::asynchronous::Client; + use tempfile::TempDir; + use wavekv::types::{Entry, Metadata}; + + const ME: u32 = 1; + const PEER: u32 = 2; + /// Both sides of the local mTLS test are this gateway, so one id serves for + /// the certificate it presents and the identity it expects of a peer. + const TEST_APP_ID: &[u8] = b"test-app-id-0000-0001"; + + fn peer_uuid() -> Vec<u8> { + b"the-real-peer-2".to_vec() + } + + /// A self-signed CA plus a leaf it signs. `HttpSyncNetwork::new` loads all three + /// from disk to build its rustls client config, and the root store only accepts a + /// trust anchor with `CA:TRUE` — so a lone self-signed leaf is not enough. + /// + /// The leaf carries an app_id because the peer check is unconditional, so a + /// certificate without one is refused before any of these tests reach what they are + /// about. It is also what a real peer's certificate carries. + fn write_tls_material(dir: &std::path::Path) -> TlsConfig { + let pki = ra_tls::test_pki::write_mtls_pki( + dir, + ra_tls::test_pki::TestCert::localhost().app_id(TEST_APP_ID), + ) + .expect("write test PKI"); + + TlsConfig { + certs: pki.cert_path.to_string_lossy().into_owned(), + key: pki.key_path.to_string_lossy().into_owned(), + mutual: MutualConfig { + ca_certs: pki.ca_cert_path.to_string_lossy().into_owned(), + }, + } + } + + /// A gateway whose request-handling half these tests drive directly. + /// + /// They are about everything below the peer check — the gzip framing, the store + /// split, the uuid check, the removed-sender refusal — and they call `handle_sync` + /// and `handle_push` rather than going through the routes, because the peer check is + /// now unconditional and Rocket's local client speaks no TLS, so a request through + /// the route can never get past it. `enforcing_gateway` covers the check itself. + async fn serving_gateway(sync_enabled: bool) -> (Proxy, TempDir) { + let (_client, proxy, tmp) = serving_gateway_with(sync_enabled).await; + (proxy, tmp) + } + + /// The same gateway, reached through the real routes, where every request is refused + /// for want of a certificate. That refusal is the assertion. + async fn enforcing_gateway() -> (Client, Proxy, TempDir) { + serving_gateway_with(true).await + } + + /// Drive the sync endpoint's body the way its route would. + fn post_sync(proxy: &Proxy, store: &str, body: Vec<u8>) -> (Status, Vec<u8>) { + match handle_sync(proxy, store, &body) { + Ok((_content_type, bytes)) => (Status::Ok, bytes), + Err(status) => (status, Vec::new()), + } + } + + /// Drive the push endpoint's body the way its route would. + fn post_push(proxy: &Proxy, store: &str, body: Vec<u8>) -> Status { + match handle_push(proxy, store, &body) { + Ok(status) => status, + Err(status) => status, + } + } + + async fn serving_gateway_with(sync_enabled: bool) -> (Client, Proxy, TempDir) { + // `main` installs this once at startup; the sync client builds a rustls config, + // so a test that skips it panics inside rustls rather than failing an assertion. + let _ = rustls::crypto::ring::default_provider().install_default(); + + let figment = load_config_figment(None); + let mut config = figment.focus("core").extract::<Config>().unwrap(); + let temp_dir = TempDir::new().expect("temp dir"); + + config.sync.enabled = sync_enabled; + config.sync.node_id = ME; + config.sync.bootnode = String::new(); + config.sync.data_dir = temp_dir.path().to_string_lossy().into_owned(); + config.wg.config_path = temp_dir + .path() + .join("wg.conf") + .to_string_lossy() + .into_owned(); + + let tls_config = write_tls_material(temp_dir.path()); + let proxy = Proxy::new(ProxyOptions { + config, + my_app_id: TEST_APP_ID.to_vec(), + tls_config, + }) + .await + .expect("failed to build gateway"); + + let rocket = rocket::build() + .manage(proxy.clone()) + .mount("/", crate::web_routes::wavekv_sync_routes()); + let client = Client::tracked(rocket).await.expect("rocket client"); + (client, proxy, temp_dir) + } + + /// The sync routes are the cluster's write surface: anything that reaches them can + /// insert entries that replicate to every gateway. `verify_gateway_peer` is the only + /// thing standing in front of them, and no test had ever executed it. Replacing the + /// whole function body with `Ok(())` did not turn the suite red. + /// + /// Rocket's local client speaks no TLS and so presents no certificate, which is + /// exactly the case that must be refused. + #[tokio::test] + async fn every_sync_route_refuses_a_peer_it_cannot_identify() { + let (client, _proxy, _tmp) = enforcing_gateway().await; + + for route in ["/wavekv/sync/persistent", "/wavekv/push/persistent"] { + let response = client.post(route).body(Vec::new()).dispatch().await; + assert_eq!( + response.status(), + Status::Unauthorized, + "{route} served a request from an unauthenticated caller" + ); + } + } + + /// A real certificate carrying `PHALA_RATLS_APP_ID`, minted locally. + /// + /// Nothing here needs a TEE: the extension is an ordinary X.509 extension that + /// `CertRequest` adds unconditionally, and the check under test never looks at a + /// quote — it reads two extensions and compares bytes. + fn cert_with_app_id(app_id: &[u8]) -> Vec<u8> { + ra_tls::test_pki::TestCert::new("peer.test") + .app_id(app_id) + .self_signed_der() + .expect("self-signed cert") + } + + /// A certificate with no app identity at all. + fn cert_without_app_id() -> Vec<u8> { + ra_tls::test_pki::TestCert::new("peer.test") + .self_signed_der() + .expect("self-signed cert") + } + + fn authorize(der: &[u8], my_app_id: &[u8]) -> Result<(), Status> { + use rocket::mtls::x509::{FromDer, X509Certificate}; + let (_, parsed) = X509Certificate::from_der(der).expect("parse cert"); + authorize_peer(&RocketCert(parsed.extensions()), my_app_id) + } - // Encode response - let encoded = encode_sync_response(&response)?; + /// The rule the sync routes are defended by: same app id or nothing. + /// + /// Every case below was previously unreachable, because the only tests that touched + /// this code returned before it without checking anything. Inverting the + /// comparison to `==` left the suite green. + #[test] + fn a_peer_is_authorized_only_when_its_app_id_matches_ours() { + let ours = b"app-id-of-this-cluster".to_vec(); - Ok((ContentType::new("application", "x-msgpack-gz"), encoded)) + assert_eq!(authorize(&cert_with_app_id(&ours), &ours), Ok(())); + + assert_eq!( + authorize(&cert_with_app_id(b"a-different-app"), &ours), + Err(Status::Forbidden), + "a valid certificate from another app must not reach the sync routes" + ); + } + + /// App info is not app id. + /// + /// This check used to fall back to the app-info extension when app-id was absent, + /// which made it the only one of the cluster's three identity checks that would + /// accept such a certificate -- `AppIdValidator` in the sync client and + /// `ensure_from_gateway` in the RPC handler both refuse it. The fallback existed to + /// paper over locally issued certificates that carried no app id; that is fixed + /// where it is caused now, in cert-client. + /// + /// Restoring the fallback turns this red. Nothing did before: the branch had no + /// test at all, and deleting it left all 308 gateway tests green. + #[test] + fn app_info_alone_does_not_authorize_a_peer() { + let ours = b"app-id-of-this-cluster".to_vec(); + let cert = ra_tls::test_pki::TestCert::new("peer.test") + .app_info(&ours) + .self_signed_der() + .expect("self-signed cert"); + + assert_eq!( + authorize(&cert, &ours), + Err(Status::Unauthorized), + "the app-id extension is the only identity the sync routes accept" + ); + } + + /// A certificate that proves nothing about which app presented it is refused, rather + /// than falling through to a comparison against `None`. + #[test] + fn a_certificate_without_an_app_id_is_refused() { + assert_eq!( + authorize(&cert_without_app_id(), b"app-id-of-this-cluster"), + Err(Status::Unauthorized) + ); + } + + /// The adapter must match the app-id extension by OID and no other. Returning some + /// other extension's bytes would hand `authorize_peer` a value it would happily + /// compare. + #[test] + fn the_adapter_reads_the_app_id_extension_and_not_a_neighbour() { + use ra_tls::traits::CertExt; + use rocket::mtls::x509::{FromDer, X509Certificate}; + + let der = cert_with_app_id(b"the-app-id"); + let (_, parsed) = X509Certificate::from_der(&der).expect("parse cert"); + let adapter = RocketCert(parsed.extensions()); + + assert_eq!( + adapter.get_app_id().expect("read app id"), + Some(b"the-app-id".to_vec()) + ); + assert_eq!( + adapter.get_special_usage().expect("read special usage"), + None, + "an extension that was never set must read back as absent" + ); + } + + /// Register the peer so `query_uuid` returns something: the uuid check is opt-in and + /// an unknown sender bypasses it entirely. + fn register_peer(proxy: &Proxy) { + proxy + .kv_store() + .sync_node( + PEER, + &NodeData { + uuid: peer_uuid(), + url: "https://peer.test:8011".to_string(), + wg_public_key: String::new(), + wg_endpoint: String::new(), + wg_ip: String::new(), + }, + ) + .expect("register peer"); + } + + fn push_envelope(uuid: Vec<u8>, key: &str) -> SyncEnvelope { + let mut env = SyncEnvelope::new(PEER, uuid); + env.push_only = true; + env.entries.push(Entry::new( + key.to_string(), + Some(b"v".to_vec()), + Metadata::new(PEER, 1, 1), + )); + env + } + + fn body(env: &SyncEnvelope) -> Vec<u8> { + gzip(&env.encode().expect("encode envelope")).expect("gzip") + } + + #[tokio::test] + async fn a_stamped_push_is_accepted_and_lands_in_the_store() { + let (proxy, _tmp) = serving_gateway(true).await; + register_peer(&proxy); + + let status = post_push( + &proxy, + "persistent", + body(&push_envelope(peer_uuid(), "node/9")), + ); + + assert_eq!(status, Status::Ok); + assert!( + proxy.kv_store().persistent().read().get("node/9").is_some(), + "a well-formed push must reach the store" + ); + } + + /// The route-level view of the bug that made every opportunistic push fail: the + /// sender built its envelope without stamping `sender_uuid`, and the receiver's + /// `check_uuid` — which only the manager runs, not `merge_push` — rejected it. + #[tokio::test] + async fn an_unstamped_push_is_refused_at_the_route() { + let (proxy, _tmp) = serving_gateway(true).await; + register_peer(&proxy); + + let status = post_push( + &proxy, + "persistent", + body(&push_envelope(Vec::new(), "node/9")), + ); + + assert_eq!(status, Status::InternalServerError); + assert!( + proxy.kv_store().persistent().read().get("node/9").is_none(), + "a push that fails the identity check must not write anything" + ); + } + + #[tokio::test] + async fn a_sync_round_trip_returns_a_decodable_envelope() { + let (proxy, _tmp) = serving_gateway(true).await; + register_peer(&proxy); + proxy + .kv_store() + .persistent() + .write() + .put("node/7".to_string(), b"v".to_vec()) + .expect("seed"); + + let request = SyncEnvelope::new(PEER, peer_uuid()); + let (status, bytes) = post_sync(&proxy, "persistent", body(&request)); + + assert_eq!(status, Status::Ok); + let decoded = SyncEnvelope::decode(&gunzip(&bytes).expect("gunzip")).expect("decode"); + assert_eq!(decoded.sender_id, ME); + assert!( + decoded.entries.iter().any(|e| e.key == "node/7"), + "an empty ack map must draw the whole live state" + ); + } + + /// Exercise the composed transport rather than testing the HTTPS client and Rocket + /// routes in isolation: a real TLS listener requires a CA-signed client certificate, + /// receives a compressed envelope, merges it, and returns a decodable response. + #[tokio::test] + async fn sync_and_push_cross_a_real_mutually_authenticated_tls_connection() { + use rocket::{mtls::MtlsConfig, tls::TlsConfig as RocketTlsConfig}; + + let (proxy, tmp) = serving_gateway(true).await; + register_peer(&proxy); + let tls = write_tls_material(tmp.path()); + + let port = { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port"); + listener.local_addr().expect("local address").port() + }; + let server_tls = RocketTlsConfig::from_paths(&tls.certs, &tls.key) + .with_mutual(MtlsConfig::from_path(&tls.mutual.ca_certs).mandatory(true)); + let figment = rocket::Config::figment() + .merge(("address", "127.0.0.1")) + .merge(("port", port)) + .merge(("tls", server_tls)); + let rocket = rocket::custom(figment) + .manage(proxy.clone()) + .mount("/", crate::web_routes::wavekv_sync_routes()) + .ignite() + .await + .expect("ignite TLS Rocket server"); + let shutdown = rocket.shutdown(); + let server = tokio::spawn(async move { + rocket.launch().await.expect("TLS Rocket server"); + }); + + let client = HttpsClient::new(&crate::kv::HttpsClientConfig { + cert_path: tls.certs.clone(), + key_path: tls.key.clone(), + ca_cert_path: tls.mutual.ca_certs.clone(), + cert_validator: std::sync::Arc::new(crate::kv::AppIdValidator::new( + TEST_APP_ID.to_vec(), + )), + }) + .expect("HTTPS client"); + let base = format!("https://127.0.0.1:{port}"); + + let mut request = SyncEnvelope::new(PEER, peer_uuid()); + request.entries.push(Entry::new( + "node/21".to_string(), + Some(b"sync".to_vec()), + Metadata::new(PEER, 21, 1), + )); + + let response = { + let mut last = None; + let mut response = None; + for _ in 0..50 { + match client + .post_bytes_response( + &format!("{base}/wavekv/sync/persistent"), + request.encode().expect("encode sync request"), + ) + .await + { + Ok(bytes) => { + response = Some(bytes); + break; + } + Err(err) => { + last = Some(err); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + } + } + response.unwrap_or_else(|| panic!("server did not become ready: {last:?}")) + }; + SyncEnvelope::decode(&response).expect("decode sync response"); + assert!(proxy + .kv_store() + .persistent() + .read() + .get("node/21") + .is_some()); + + let push = push_envelope(peer_uuid(), "node/22"); + client + .post_bytes_no_response( + &format!("{base}/wavekv/push/persistent"), + push.encode().expect("encode push"), + ) + .await + .expect("push over mTLS"); + assert!(proxy + .kv_store() + .persistent() + .read() + .get("node/22") + .is_some()); + + shutdown.notify(); + server.await.expect("server task"); + } + + /// The production routes with the peer check removed. + /// + /// Mounted by one test, and only because the body cap it asserts lives in + /// the route half: `handle_sync` never sees a `Data`, so a test calling it + /// directly cannot reach the limit, and Rocket's local client cannot present + /// the certificate the real route now requires. Two duplicated lines are + /// cheaper than losing coverage of a bound that exists to stop a peer -- or a + /// stranger who got that far -- from making the gateway read without end. + #[post("/wavekv/sync/<store>", data = "<data>")] + async fn ungated_sync( + state: &State<Proxy>, + store: &str, + data: Data<'_>, + ) -> Result<(ContentType, Vec<u8>), Status> { + let body = read_compressed_body(data).await?; + handle_sync(state, store, &body) + } + + #[post("/wavekv/push/<store>", data = "<data>")] + async fn ungated_push( + state: &State<Proxy>, + store: &str, + data: Data<'_>, + ) -> Result<Status, Status> { + let body = read_compressed_body(data).await?; + handle_push(state, store, &body) + } + + #[tokio::test] + async fn an_oversized_compressed_request_is_rejected_explicitly() { + let (proxy, _tmp) = serving_gateway(true).await; + let rocket = rocket::build() + .manage(proxy) + .mount("/", rocket::routes![ungated_sync, ungated_push]); + let client = Client::tracked(rocket).await.expect("rocket client"); + + for path in ["/wavekv/sync/persistent", "/wavekv/push/persistent"] { + let response = client + .post(path) + .body(vec![0u8; 16 * 1024 * 1024 + 1]) + .dispatch() + .await; + assert_eq!(response.status(), Status::PayloadTooLarge, "{path}"); + } + } + + /// Unknown stores are rejected rather than being routed to either replicated store. + #[tokio::test] + async fn an_unknown_store_is_rejected() { + let (proxy, _tmp) = serving_gateway(true).await; + register_peer(&proxy); + + let (status, _) = post_sync(&proxy, "bogus", body(&SyncEnvelope::new(PEER, peer_uuid()))); + + assert_eq!(status, Status::NotFound); + } + + /// The lockout at the door. The app-identity check proves the sender is + /// *a* gateway of this app; only the removal marker says whether it is + /// still a *member*. A removed node's envelopes are refused on both + /// routes before anything merges, and clearing the marker -- the + /// SetNodeUrl re-admission path -- opens the door again. + #[tokio::test] + async fn a_removed_nodes_envelopes_are_refused_at_the_door() { + let (proxy, _tmp) = serving_gateway(true).await; + register_peer(&proxy); + proxy.kv_store().mark_peer_removed(PEER).expect("mark"); + + let (sync_status, _) = post_sync( + &proxy, + "persistent", + body(&SyncEnvelope::new(PEER, peer_uuid())), + ); + assert_eq!( + sync_status, + Status::Forbidden, + "sync must refuse a removed sender" + ); + let push_status = post_push( + &proxy, + "persistent", + body(&SyncEnvelope::new(PEER, peer_uuid())), + ); + assert_eq!( + push_status, + Status::Forbidden, + "push must refuse a removed sender" + ); + + proxy.kv_store().clear_peer_removed(PEER).expect("clear"); + let (readmitted, _) = post_sync( + &proxy, + "persistent", + body(&SyncEnvelope::new(PEER, peer_uuid())), + ); + assert_eq!(readmitted, Status::Ok, "re-admission opens the door again"); + } + + /// A node with synchronization disabled reports that the service is unavailable. + #[tokio::test] + async fn a_sync_disabled_node_answers_503() { + let (proxy, _tmp) = serving_gateway(false).await; + + let (sync_status, _) = post_sync( + &proxy, + "persistent", + body(&SyncEnvelope::new(PEER, peer_uuid())), + ); + assert_eq!( + sync_status, + Status::ServiceUnavailable, + "sync must not look like a missing v2 route" + ); + let push_status = post_push( + &proxy, + "persistent", + body(&SyncEnvelope::new(PEER, peer_uuid())), + ); + assert_eq!( + push_status, + Status::ServiceUnavailable, + "push must not look like a missing v2 route" + ); + } + + /// gzip expands by three orders of magnitude on attacker-chosen input, so the + /// 16 MiB cap on the request body bounds the *compressed* size and nothing else. + /// mTLS proves only that the sender is some gateway of this deployment, so the bound + /// has to hold against a peer running a buggy build, not just against a stranger. + #[tokio::test] + async fn a_compression_bomb_is_refused_before_it_is_decompressed() { + let (proxy, _tmp) = serving_gateway(true).await; + + // ~130 MiB of zeroes compresses to well under the request cap. + let bomb = gzip(&vec![0u8; MAX_DECOMPRESSED_SYNC_BYTES + 1]).expect("gzip"); + assert!( + bomb.len() < 16 * 1024 * 1024, + "the fixture has to fit through the body cap to be testing anything: {} bytes", + bomb.len() + ); + + let (sync_status, _) = post_sync(&proxy, "persistent", bomb.clone()); + assert_eq!( + sync_status, + Status::BadRequest, + "sync must refuse an over-sized expansion" + ); + let push_status = post_push(&proxy, "persistent", bomb); + assert_eq!( + push_status, + Status::BadRequest, + "push must refuse an over-sized expansion" + ); + } + + /// The limits must leave room for the largest legitimate message. + /// + /// The boundary test below asserts a payload of exactly `MAX_DECOMPRESSED_SYNC_BYTES` + /// is accepted — but it builds that payload *from the same constant*, so it holds + /// whatever the constant says. Shrinking the limit to a few kilobytes keeps it green + /// while rejecting every real delta. Pin the values against what production sends, + /// which is the property that actually matters. + // Deliberately runtime assertions rather than `const { assert!(..) }`: a const block + // would fail the build, which mutation testing scores as "unviable" rather than + // "caught", and would lose the message explaining what the number is for. + #[allow(clippy::assertions_on_constants)] + #[test] + fn the_sync_limits_admit_the_largest_message_the_protocol_can_produce() { + // A v2 delta is capped by wavekv's `max_delta_bytes` (4 MiB by default). + const MAX_DELTA_BYTES: usize = 4 * 1024 * 1024; + assert!( + MAX_DECOMPRESSED_SYNC_BYTES >= 8 * MAX_DELTA_BYTES, + "a decompression limit of {MAX_DECOMPRESSED_SYNC_BYTES} bytes would reject \ + ordinary sync traffic, not just a bomb" + ); + + // The compressed ceiling mirrors what the routes accept on a request, so a peer + // cannot answer with more than it would have been allowed to ask. + assert_eq!( + crate::kv::MAX_COMPRESSED_SYNC_BYTES, + 16 * 1024 * 1024, + "this must stay equal to the 16 MiB the routes accept on a request body" + ); + } + + /// The limit is inclusive, so a payload landing exactly on it still decodes. Without + /// this the bound could tighten by a byte and only the bomb test would still pass. + #[test] + fn a_payload_exactly_on_the_limit_still_decompresses() { + let exact = gzip(&vec![7u8; MAX_DECOMPRESSED_SYNC_BYTES]).expect("gzip"); + let out = gunzip_bounded(&exact, MAX_DECOMPRESSED_SYNC_BYTES).expect("must be accepted"); + assert_eq!(out.len(), MAX_DECOMPRESSED_SYNC_BYTES); + + let one_over = gzip(&vec![7u8; MAX_DECOMPRESSED_SYNC_BYTES + 1]).expect("gzip"); + assert!(gunzip_bounded(&one_over, MAX_DECOMPRESSED_SYNC_BYTES).is_err()); + } + + #[tokio::test] + async fn a_push_from_node_id_zero_is_refused() { + let (proxy, _tmp) = serving_gateway(true).await; + register_peer(&proxy); + + let mut env = push_envelope(peer_uuid(), "node/9"); + env.sender_id = 0; + let status = post_push(&proxy, "persistent", body(&env)); + + assert_eq!(status, Status::BadRequest); + } } diff --git a/dstack/gateway/templates/dashboard.html b/dstack/gateway/templates/dashboard.html index 55a296ff8..aa43718dc 100644 --- a/dstack/gateway/templates/dashboard.html +++ b/dstack/gateway/templates/dashboard.html @@ -182,6 +182,20 @@ color: #f44336; } + .traffic-gated { + color: #f44336; + font-weight: bold; + } + + .health-unhealthy { + color: #f44336; + font-weight: bold; + } + + .health-unknown { + color: #999; + } + .status-controls { display: flex; gap: 5px; @@ -452,6 +466,51 @@ <h2>This Node</h2> </tr> </table> + <table> + <tr> + <th colspan="3">Data Path</th> + </tr> + <tr> + <td>kTLS</td> + <td>{{ accel.ktls_mode }}</td> + <td>Record encryption in the kernel. Cleared at startup if the + kernel has no TLS ULP.</td> + </tr> + <tr> + <td>Splice</td> + <td>{{ accel.splice_mode }}</td> + <td>Zero-copy relaying, once a connection trips the gate.</td> + </tr> + <tr> + <td>kTLS Offloads</td> + <td>{{ accel.ktls_offloaded }}</td> + <td>Connections handed to the kernel since start.</td> + </tr> + <tr> + <td>kTLS Offload Failures</td> + <td>{{ accel.ktls_offload_failed }}</td> + <td>Non-zero means connections are failing at the handover, not + merely running unaccelerated.</td> + </tr> + <tr> + <td>Splice Engagements</td> + <td>{{ accel.splice_engaged }}</td> + <td>Connections that entered a zero-copy relay, on either path.</td> + </tr> + <tr> + <td>Health Gating</td> + {% if status.health_gating %} + <td>on</td> + <td>Instances that opted in are held out of app-id load balancing + until a poll says the app is serving.</td> + {% else %} + <td class="health-unhealthy">off</td> + <td>Nothing is polled and every instance is eligible, whatever the + Health column says.</td> + {% endif %} + </tr> + </table> + <h2>Global Connections</h2> <div id="global-connections"> <p>Loading global connection statistics...</p> @@ -463,7 +522,7 @@ <h2>Global Connections</h2> </tr> <tr> <td>Account URI</td> - <td>{{ acme_info.account_uri }}</td> + <td id="acme-account-uri">{{ acme_info.account_uri }}</td> </tr> <tr> <td>Historical Certificate Public Keys</td> @@ -490,6 +549,24 @@ <h2>Certbot Configuration</h2> </td> <td>ACME server URL (empty = Let's Encrypt production)</td> </tr> + <tr> + <td>ACME Account</td> + <td> + <button class="action-btn warning" onclick="rotateAcmeCredentials()">Register / Rotate</button> + </td> + <td> + Registers the shared ACME account. Every run registers a <em>new</em> one, so + <code>dns-persist-01</code> validation records and account-pinned CAA have to be + republished afterwards. + </td> + </tr> + <tr> + <td>Issuer Domain Name</td> + <td> + <input type="text" id="certbot-issuer-domain-name" placeholder="letsencrypt.org (default)"> + </td> + <td>Name of the CA in dns-persist-01 and CAA records; must be one the ACME server advertises</td> + </tr> <tr> <td>Renewal Interval</td> <td> @@ -576,6 +653,8 @@ <h2>CVM List</h2> <th>IP</th> <th>Last Seen</th> <th>Connections</th> + <th>Health</th> + <th>Traffic</th> </tr> {% for host in status.hosts %} <tr> @@ -584,6 +663,24 @@ <h2>CVM List</h2> <td>{{ host.ip }}</td> <td class="timestamp">{{ host.latest_handshake }}</td> <td>{{ host.num_connections }}</td> + {% if host.health == "unhealthy" %} + <td class="health-unhealthy" title="The guest agent reports the app is not serving.">unhealthy</td> + {% else if host.health == "unknown" %} + {% if status.health_gating %} + <td class="health-unknown" title="Registered, but no health poll has answered yet. Held out of rotation until one does.">unknown</td> + {% else %} + <td class="health-unknown" title="This app asked to be gated on its health, but polling is off on this node (core.proxy.health_check.enabled). Nothing will ever answer, and the instance is in rotation.">unknown (gating off)</td> + {% endif %} + {% else if host.health == "ungated" %} + <td class="health-unknown" title="This app did not set requirements.health_check, so it is never polled and always eligible.">not polled</td> + {% else %} + <td>{{ host.health }}</td> + {% endif %} + {% if host.ready.unwrap_or(true) %} + <td>in rotation</td> + {% else %} + <td class="traffic-gated" title="Removed from app-id load balancing by an operator. Still reachable by instance id.">gated off</td> + {% endif %} </tr> {% endfor %} </table> @@ -681,6 +778,13 @@ <h3>Edit ZT-Domain</h3> <option value="">Use Default</option> </select> </div> + <div class="modal-field"> + <label>Validation Challenge</label> + <select id="edit-domain-challenge"> + <option value="dns-01">dns-01 (certbot writes DNS; needs a credential)</option> + <option value="dns-persist-01">dns-persist-01 (one record published by hand; experimental)</option> + </select> + </div> <div class="modal-field"> <label>Port (0 = default)</label> <input type="number" id="edit-domain-port" placeholder="443" min="0" max="65535"> @@ -700,6 +804,39 @@ <h3>Edit ZT-Domain</h3> </div> </div> + <!-- DNS Records To Publish Modal --> + <div id="dns-records-modal" class="modal-overlay"> + <div class="modal"> + <h3>What is left to do</h3> + <div id="dns-records-repin" style="display: none;"> + <p style="color: #b00;"> + The rotation succeeded and the cluster is on the new account, but these + domains' CAA records still name the old one, so nothing issues for them + until a <code>SetCaa</code> run takes. Re-running the rotation would + register yet another account against a rate-limited quota. + </p> + <pre id="dns-records-repin-list" style="white-space: pre-wrap; word-break: break-all;"></pre> + <div class="modal-actions"> + <button class="action-btn warning" onclick="retryCaa()">Run SetCaa</button> + </div> + </div> + <div id="dns-records-publish"> + <p style="color: #666;"> + This domain validates with <code>dns-persist-01</code>, so the gateway holds no + DNS credential for it and cannot write these itself. Certificates are only + issued once they are published. + </p> + <pre id="dns-records-list" style="white-space: pre-wrap; word-break: break-all;"></pre> + <div class="modal-actions"> + <button class="action-btn secondary" onclick="copyDnsRecords()">Copy</button> + </div> + </div> + <div class="modal-actions"> + <button class="action-btn primary" onclick="hideModal('dns-records-modal')">Done</button> + </div> + </div> + </div> + <!-- Add ZT-Domain Modal --> <div id="add-zt-domain-modal" class="modal-overlay"> <div class="modal"> @@ -714,6 +851,13 @@ <h3>Add ZT-Domain</h3> <option value="">Use Default</option> </select> </div> + <div class="modal-field"> + <label>Validation Challenge</label> + <select id="domain-challenge"> + <option value="dns-01">dns-01 (certbot writes DNS; needs a credential)</option> + <option value="dns-persist-01">dns-persist-01 (one record published by hand; experimental)</option> + </select> + </div> <div class="modal-field"> <label>Port</label> <input type="number" id="domain-port" placeholder="443" min="0" max="65535" value="443"> @@ -814,7 +958,6 @@ <h3>Add ZT-Domain</h3> <th>Peer ID</th> <th>Local Ack</th> <th>Peer Ack</th> - <th>Buffered</th> <th>Last Seen</th> </tr> `; @@ -830,14 +973,12 @@ <h3>Add ZT-Domain</h3> : 'N/A'; const localAck = peer.local_ack || peer.localAck || 0; const peerAck = peer.peer_ack || peer.peerAck || 0; - const bufferedLogs = peer.buffered_logs || peer.bufferedLogs || 0; html += ` <tr> <td class="truncated-id" title="${peer.id}">${peer.id}</td> <td>${localAck}</td> <td>${peerAck}</td> - <td>${bufferedLogs}</td> <td class="last-seen-cell">${lastSeenStr}</td> </tr> `; @@ -876,6 +1017,7 @@ <h3>Add ZT-Domain</h3> }); const data = await response.json(); document.getElementById('certbot-acme-url').value = data.acme_url || ''; + document.getElementById('certbot-issuer-domain-name').value = data.issuer_domain_name || ''; document.getElementById('certbot-renew-interval').value = data.renew_interval_secs || 43200; document.getElementById('certbot-renew-before').value = data.renew_before_expiration_secs || 2592000; document.getElementById('certbot-renew-timeout').value = data.renew_timeout_secs || 300; @@ -886,6 +1028,7 @@ <h3>Add ZT-Domain</h3> async function saveCertbotConfig() { const acmeUrl = document.getElementById('certbot-acme-url').value.trim(); + const issuerDomainName = document.getElementById('certbot-issuer-domain-name').value.trim(); const renewInterval = parseInt(document.getElementById('certbot-renew-interval').value) || undefined; const renewBefore = parseInt(document.getElementById('certbot-renew-before').value) || undefined; const renewTimeout = parseInt(document.getElementById('certbot-renew-timeout').value) || undefined; @@ -893,6 +1036,7 @@ <h3>Add ZT-Domain</h3> try { const body = {}; if (acmeUrl !== undefined) body.acme_url = acmeUrl; + if (issuerDomainName !== undefined) body.issuer_domain_name = issuerDomainName; if (renewInterval !== undefined) body.renew_interval_secs = renewInterval; if (renewBefore !== undefined) body.renew_before_expiration_secs = renewBefore; if (renewTimeout !== undefined) body.renew_timeout_secs = renewTimeout; @@ -1151,6 +1295,7 @@ <h3>Add ZT-Domain</h3> <th>Port</th> <th>Node</th> <th>Priority</th> + <th>Challenge</th> <th>DNS Credential</th> <th>Status</th> <th>Expires</th> @@ -1163,7 +1308,12 @@ <h3>Add ZT-Domain</h3> const certStatus = item.cert_status || {}; const hasCert = certStatus.has_cert; const expiresAt = certStatus.not_after ? new Date(certStatus.not_after * 1000).toLocaleDateString() : 'N/A'; - const dnsCredName = getDnsCredName(config.dns_cred_id); + const challenge = config.challenge || 'dns-01'; + // A dns-persist-01 domain holds no credential by design, so + // naming one here would read as a misconfiguration. + const dnsCredName = challenge === 'dns-persist-01' + ? 'None (published by hand)' + : getDnsCredName(config.dns_cred_id); const portDisplay = config.port ? config.port : '-'; const nodeDisplay = (config.node !== undefined && config.node !== null) ? `Node ${config.node}` : 'Any'; const priorityDisplay = config.priority || 0; @@ -1174,6 +1324,7 @@ <h3>Add ZT-Domain</h3> <td>${portDisplay}</td> <td>${nodeDisplay}</td> <td>${priorityDisplay}</td> + <td>${escapeHtml(challenge)}</td> <td>${escapeHtml(dnsCredName)}</td> <td> <div class="cert-status"> @@ -1186,6 +1337,7 @@ <h3>Add ZT-Domain</h3> <td class="timestamp" data-original="${expiresAt}">${certStatus.not_after || 0}</td> <td> <button class="action-btn secondary" onclick="showEditZtDomainModal('${escapeHtml(config.domain)}')">Edit</button> + ${challenge === 'dns-persist-01' ? `<button class="action-btn secondary" onclick="showZtDomainRecords('${escapeHtml(config.domain)}')">Records</button>` : ''} <button class="action-btn warning" onclick="renewZtDomainCert('${escapeHtml(config.domain)}', false)">Renew</button> <button class="action-btn secondary" onclick="renewZtDomainCert('${escapeHtml(config.domain)}', true)">Force Renew</button> <button class="action-btn danger" onclick="deleteZtDomain('${escapeHtml(config.domain)}')">Delete</button> @@ -1221,6 +1373,10 @@ <h3>Add ZT-Domain</h3> document.getElementById('domain-port').value = '443'; document.getElementById('domain-node').value = '0'; document.getElementById('domain-priority').value = '0'; + // Reset with the rest: a form whose every visible field was blanked + // reads as fresh, and a challenge left over from the last add is + // sent unconditionally. + document.getElementById('domain-challenge').value = 'dns-01'; showModal('add-zt-domain-modal'); } @@ -1246,6 +1402,7 @@ <h3>Add ZT-Domain</h3> document.getElementById('edit-domain-port').value = config.port || 0; document.getElementById('edit-domain-node').value = (config.node !== undefined && config.node !== null) ? config.node : 0; document.getElementById('edit-domain-priority').value = config.priority || 0; + document.getElementById('edit-domain-challenge').value = config.challenge || 'dns-01'; updateEditDnsCredSelect(); document.getElementById('edit-domain-dns-cred').value = config.dns_cred_id || ''; @@ -1269,7 +1426,11 @@ <h3>Add ZT-Domain</h3> const body = { domain: domain, port: port, - priority: priority + priority: priority, + // UpdateZtDomain replaces the whole record and an omitted + // challenge decodes as dns-01, so editing anything else would + // silently downgrade a dns-persist-01 domain. + challenge: document.getElementById('edit-domain-challenge').value }; // Only include dns_cred_id if not empty (empty means use default) if (dnsCredId) { @@ -1313,7 +1474,9 @@ <h3>Add ZT-Domain</h3> const body = { domain: domain, port: port, - priority: priority + priority: priority, + // Chosen once, at creation; the edit form carries it forward. + challenge: document.getElementById('domain-challenge').value }; // Only include dns_cred_id if not empty (empty means use default) if (dnsCredId) { @@ -1333,9 +1496,25 @@ <h3>Add ZT-Domain</h3> const err = await response.text(); throw new Error(err); } + const added = await response.json(); hideModal('add-zt-domain-modal'); showToast('ZT-Domain added', 'success'); await loadZtDomains(); + // Only for dns-persist-01: `required_dns_records` is non-empty + // for dns-01 too -- the CAA lines the gateway writes itself -- + // and this dialog says the operator has to publish them. + if (body.challenge === 'dns-persist-01') { + const records = added.required_dns_records || []; + if (records.length === 0) { + // Empty until the cluster has an ACME account: every + // record names it. Saying so beats a success toast and + // no dialog, because under this method the records the + // operator publishes are the whole setup. + showToast('No records yet \u2014 register an ACME account first', 'error'); + } else { + showRequiredDnsRecords(records); + } + } } catch (e) { showToast('Failed to add ZT-Domain: ' + e.message, 'error'); } @@ -1393,6 +1572,103 @@ <h3>Add ZT-Domain</h3> // ==================== UI Helpers ==================== + async function rotateAcmeCredentials() { + const warning = [ + 'Register a new ACME account?', + '', + 'This registers a new account at the CA, which is rate-limited, and rewrites the CAA records of every dns-01 domain to name it. Rewriting installs a temporary ";" guard and deletes the old records first, so a DNS provider failure part-way through can leave a domain with CAA that blocks all issuance until a later SetCaa succeeds.', + '', + 'dns-persist-01 domains keep naming the old account until you republish their records by hand.' + ].join('\n'); + if (!confirm(warning)) { + return; + } + try { + const response = await fetch('/prpc/Admin.RotateAcmeCredentials', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: '{}' + }); + if (!response.ok) { + throw new Error(await response.text()); + } + const result = await response.json(); + // The account URI is server-rendered, so it would otherwise keep + // showing the account this rotation just replaced. + document.getElementById('acme-account-uri').textContent = result.account_uri; + showToast('ACME account: ' + result.account_uri, 'success'); + await loadZtDomains(); + // Everything still outstanding goes in the dialog, not a toast: + // a toast is gone in three seconds and the modal below would + // cover it, leaving a page that reads as a clean rotation. The + // response is the only place either list is reported. + showRequiredDnsRecords(result.required_dns_records, result.repin_failed_domains); + } catch (e) { + showToast('Failed to register ACME account: ' + e.message, 'error'); + } + } + + function showZtDomainRecords(domain) { + const item = ztDomains.find(d => d.config && d.config.domain === domain); + const records = item && item.required_dns_records; + if (!records || records.length === 0) { + // Empty until the cluster has an ACME account: every record + // names it. RotateAcmeCredentials registers one. + showToast('No records yet — register an ACME account first', 'error'); + return; + } + showRequiredDnsRecords(records); + } + + // Both halves of "this operation is not finished" in one dialog: the + // records only an operator can publish, and the domains whose CAA the + // gateway tried to re-pin and could not. A dns-01-only cluster has the + // second without the first, so either one alone still opens it. + function showRequiredDnsRecords(records, repinFailed) { + records = records || []; + repinFailed = repinFailed || []; + if (records.length === 0 && repinFailed.length === 0) { + return; + } + document.getElementById('dns-records-publish').style.display = + records.length > 0 ? '' : 'none'; + document.getElementById('dns-records-list').textContent = records.join('\n'); + document.getElementById('dns-records-repin').style.display = + repinFailed.length > 0 ? '' : 'none'; + document.getElementById('dns-records-repin-list').textContent = repinFailed.join('\n'); + showModal('dns-records-modal'); + } + + async function retryCaa() { + try { + const response = await fetch('/prpc/Admin.SetCaa', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: '{}' + }); + if (!response.ok) { + throw new Error(await response.text()); + } + // SetCaa raises on partial failure, so reaching here means every + // domain took. Unlike rotation it is idempotent, which is why + // this is the operation to repeat and rotation is not. + document.getElementById('dns-records-repin').style.display = 'none'; + showToast('CAA re-pinned', 'success'); + } catch (e) { + showToast('SetCaa failed, run it again: ' + e.message, 'error'); + } + } + + async function copyDnsRecords() { + const text = document.getElementById('dns-records-list').textContent; + try { + await navigator.clipboard.writeText(text); + showToast('Records copied', 'success'); + } catch (e) { + showToast('Copy failed: ' + e.message, 'error'); + } + } + function showModal(id) { document.getElementById(id).classList.add('active'); } diff --git a/dstack/gateway/test-run/.env.example b/dstack/gateway/test-run/.env.example deleted file mode 100644 index ff6571750..000000000 --- a/dstack/gateway/test-run/.env.example +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> -# -# SPDX-License-Identifier: Apache-2.0 - -# Cloudflare API token with DNS edit permissions -# Required scopes: Zone.DNS (Edit), Zone.Zone (Read) -CF_API_TOKEN=your_cloudflare_api_token_here - -# Cloudflare Zone ID for your domain -CF_ZONE_ID=your_zone_id_here - -# Test domain (must be a wildcard domain managed by Cloudflare) -# Example: *.test.example.com -TEST_DOMAIN=*.test.example.com diff --git a/dstack/gateway/test-run/.gitignore b/dstack/gateway/test-run/.gitignore index 25d3f97c9..ba4042005 100644 --- a/dstack/gateway/test-run/.gitignore +++ b/dstack/gateway/test-run/.gitignore @@ -1,3 +1,12 @@ /run/ .env /e2e/dstack-gateway +/cluster/dstack-gateway +/cluster/run/ +__pycache__/ +/proxy-e2e/dstack-gateway +/proxy-e2e/test_proxy.sh +/proxy-e2e/proxy/ +/proxy-e2e/run/ +*.lock +.suite.lock diff --git a/dstack/gateway/test-run/Dockerfile.gateway b/dstack/gateway/test-run/Dockerfile.gateway new file mode 100644 index 000000000..66cc90769 --- /dev/null +++ b/dstack/gateway/test-run/Dockerfile.gateway @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +# Runtime image for a gateway built as a static musl binary. Shared by the +# gateway test suites, which each copy their own `dstack-gateway` next to their +# compose file before building. +FROM alpine:latest + +RUN apk add --no-cache \ + wireguard-tools \ + iproute2 \ + curl \ + ca-certificates + +COPY dstack-gateway /usr/local/bin/dstack-gateway + +RUN chmod +x /usr/local/bin/dstack-gateway && \ + mkdir -p /etc/gateway/certs /var/lib/gateway + +ENTRYPOINT ["/usr/local/bin/dstack-gateway", "-c", "/etc/gateway/gateway.toml"] diff --git a/dstack/gateway/test-run/TESTING.md b/dstack/gateway/test-run/TESTING.md index 0ff3269fc..945d13ec5 100644 --- a/dstack/gateway/test-run/TESTING.md +++ b/dstack/gateway/test-run/TESTING.md @@ -2,190 +2,107 @@ # # SPDX-License-Identifier: Apache-2.0 -# Gateway test plan +# Gateway integration tests -This document records the local checks used for the gateway handshake-cache -change. The goal is to verify three things: +Three suites, all under docker compose, all driven from the host. None of them +needs root on the machine running them, and none of them leaves anything behind. -1. the generic `cached-cell` crate behaves correctly; -2. existing gateway control-plane and WaveKV flows still work; -3. the proxy data path does not call blocking `wg show latest-handshakes` per - request. +| Suite | Covers | Run it | +| --- | --- | --- | +| `e2e/` | certbot, ACME, DNS-01 and `dns-persist-01` | `e2e/run-e2e.sh` | +| `cluster/` | WaveKV replication, node identity, partition recovery, admin RPCs | `cluster/run-cluster-tests.sh` | +| `proxy-e2e/` | the proxy data path: splice, kTLS, half-close, idle reaping | `proxy-e2e/run-proxy-tests.sh` | -## Prerequisites +Each has a `--skip-build` flag that reuses the `dstack-gateway` binary already +staged next to its compose file, and a `down` subcommand that tears everything +down. -Run on Linux with: +## Attestation is on -- Rust toolchain; -- `sudo`, `ip`, `wg` / `wireguard-tools`; -- `curl`, `openssl`, `python3`; -- `wrk` for the performance test. +There is no switch to turn the checks off. The gateways obtain their +RPC certificates from the guest agent simulator, which signs quotes under trust +anchors derived from `attestation/tee-simulator.json`; the mock collateral +service reconstructs the matching public roots from the same seed, and the +gateways verify each other through the normal production path. -The integration script creates temporary WireGuard interfaces named -`wavekv-test1`, `wavekv-test2`, and `wavekv-test3`, so it needs root privileges. +That is the point of the arrangement rather than a detail of it: the cluster's +mTLS is what `cluster/` exists to exercise, and a suite that switched the checks +off would be testing the switch. -## Unit and build checks +## One seed, one fixture per suite -From the repository root: +`attestation/fixture.yml` describes the simulator and the collateral service; +each suite brings up its own copy under its own `FIXTURE_NS`, and tears down +only that copy. -```bash -cargo test -p cached-cell -cargo test --manifest-path gateway/Cargo.toml -cargo check --manifest-path gateway/Cargo.toml -cargo clippy -- \ - -D warnings \ - -D clippy::expect_used \ - -D clippy::unwrap_used \ - --allow unused_variables -``` +The seed that signs the quotes and the seed that derives the verifying roots +must not drift apart, and they cannot: both come from files in `attestation/`, +so separate instances started from them agree by construction. Sharing one +running instance across the suites was the earlier arrangement and bought +nothing the files did not already guarantee -- while costing a teardown race, +where whichever suite finished first removed a fixture the others were still +using, and, in CI, three workflows that could not run at the same time. -Expected result: all commands pass. +## One project per test -## WaveKV / gateway integration test +The cluster suite gives each test a throwaway compose project of its own. Its +containers, its logs and its data directory are empty when it starts because +they are new -- not because something cleaned them up. -Build the gateway binary first: +That is the point rather than a detail. The version before it shared three +containers across all 28 tests and cleaned between them, which needed a wipe +helper, a time window on `docker logs`, and a lock to stop two runs from +clearing each other's state. Each of those three was the direct cause of a bug +during development, and each stops being possible here. -```bash -cargo build --release --manifest-path gateway/Cargo.toml -``` +## Unit and build checks -Then run the integration suite: +From `dstack/`: ```bash -cd gateway/test-run -sudo -E GATEWAY_BIN="$(pwd)/../../target/release/dstack-gateway" ./test_suite.sh -``` - -The suite starts real gateway processes and exercises: - -- CVM registration through `POST /prpc/RegisterCvm` on the debug service; -- admin RPCs such as `Admin.SetNodeUrl`, `Admin.SetNodeStatus`, and - `Admin.WaveKvStatus`; -- WaveKV persistent and ephemeral sync between gateway nodes; -- node restart, network partition recovery, periodic persistence, and node - up/down filtering. - -Expected result: - -```text -Tests passed: 19 +cargo test -p cached-cell +cargo test --manifest-path gateway/Cargo.toml +cargo clippy -- -D warnings -D clippy::expect_used -D clippy::unwrap_used \ + --allow unused_variables ``` -Important request paths covered by this suite: +## What runs in CI -| Path | Purpose | +| Workflow | Suite | | --- | --- | -| `POST /prpc/RegisterCvm` | Register a CVM, allocate a WireGuard IP, update gateway state. | -| `POST /prpc/Debug.Info` | Verify the debug service is available. | -| `POST /prpc/Debug.GetSyncData` | Inspect peer/node/instance data synced through WaveKV. | -| `POST /prpc/GetProxyState` | Compare in-memory proxy state with WaveKV state. | -| `POST /prpc/Admin.SetNodeUrl` | Register peer gateway URLs. | -| `POST /prpc/Admin.SetNodeStatus` | Mark nodes up/down and verify registration filtering. | -| `POST /prpc/Admin.WaveKvStatus` | Inspect WaveKV store status. | -| `POST /wavekv/sync/persistent` | Gateway-to-gateway persistent data sync. | -| `POST /wavekv/sync/ephemeral` | Gateway-to-gateway last-seen/handshake/connection sync. | - -## Real proxy data-path smoke test - -The integration suite above validates registration and sync, but it does not -open a client connection through the gateway proxy. For the proxy data path, use -this shape: - -1. Start one `dstack-gateway` with debug/admin enabled and `insecure_skip_attestation = true`. -2. Register a test CVM through the debug `RegisterCvm` RPC. -3. Bind a local HTTPS backend to the allocated CVM IP, for example - `10.0.51.2:23143`. -4. Serve a local DNS TXT response: - - ```text - _dstack-app-address.proxy-flow.local TXT "proxyflow:23143" - ``` - -5. Allow the backend port with `Admin.SetInstancePortPolicy` so the proxy data - path is not blocked by port-policy fail-close. -6. Send a request through the proxy: - - ```bash - curl -skf \ - --connect-to proxy-flow.local:13114:127.0.0.1:13114 \ - https://proxy-flow.local:13114/proxy-e2e - ``` - -Expected response from the backend: +| `.github/workflows/gateway-e2e-tests.yml` | `e2e/` | +| `.github/workflows/gateway-cluster-tests.yml` | `cluster/` | +| `.github/workflows/gateway-proxy-tests.yml` | `proxy-e2e/` | -```text -proxy-e2e-ok path=/proxy-e2e +```bash +gh pr checks <PR_NUMBER> --repo Dstack-TEE/dstack --watch=false ``` -Expected gateway log shape: - -```text -got sni: proxy-flow.local -target address is proxyflow:23143 -connecting to 10.0.51.2:23143 -connected to 10.0.51.2:23143 -``` +## The kTLS fallback arm -This confirms the real data flow: +One arm of the proxy suite asserts that a gateway configured for kTLS on a +kernel without the TLS ULP falls back to userspace instead of truncating a gated +transfer at the gate. It runs in its own container, because the condition is +produced by a seccomp profile -- `setsockopt(IPPROTO_TCP, TCP_ULP)` returns +`ENOPROTOOPT` -- and a seccomp profile is fixed when a container is created. -```text -client -> gateway proxy -> SNI parse -> DNS TXT lookup -> ProxyState selection -> backend TLS service -``` +See `proxy-e2e/README.md` for why that replaced taking the module away from the +host with `rmmod`. ## Proxy performance / hot-path check -The performance test uses the same real proxy data flow as the smoke test, with -one extra control: put a temporary `wg` wrapper earlier in `PATH` for the gateway -process. The wrapper delegates normal commands to `/usr/bin/wg`, but for - -```text -wg show <iface> latest-handshakes -``` - -it returns a fixed test public key and records the call. This verifies that the -proxy hot path does not execute blocking `wg show` for every request. - -Use `wrk` for three measurements: - -```bash -# Direct backend baseline. -wrk -t4 -c64 -d15s https://10.0.62.2:23243/bench - -# Gateway proxy with keep-alive. -wrk -t4 -c64 -d15s https://proxy-perf.local:13214/bench - -# Gateway proxy with new TLS connections. -wrk -t4 -c32 -d10s -H 'Connection: close' \ - https://proxy-perf.local:13214/bench-close -``` - -Reference result from the local PR run: +Not part of any suite, and not reproducible from this tree: the `wg` wrapper and +the `/bench` endpoints it describes were never checked in. The numbers below are +kept as the record of one manual run made for the handshake-cache change, and +the claim they support -- that the proxy hot path does not shell out to +`wg show latest-handshakes` per request -- is what the assertion in +`test_proxy.sh` (`test_accel_status`) covers on every run. ```text direct backend keep-alive: 71507 req/s, avg latency 1.14ms gateway proxy keep-alive: 33842 req/s, avg latency 8.83ms gateway proxy connection-close: 874 req/s, avg latency 33.45ms -``` - -The same run handled more than 500k proxy keep-alive requests. The `wg` wrapper -recorded: -```text -wg show latest-handshakes: 7 +wg show latest-handshakes: 7 (over 500k proxied keep-alive requests) wg syncconf: 3 ``` - -The important assertion is the call count: `wg show latest-handshakes` is only -used by startup/preload and the periodic refresh task, not once per proxied -request. - -## PR CI - -Check GitHub Actions before merging: - -```bash -gh pr checks <PR_NUMBER> --repo Dstack-TEE/dstack --watch=false -``` - -Expected result: all required checks pass, including `gateway`, `rust-checks`, -`prek`, `reuse-lint`, and CodeQL. diff --git a/dstack/gateway/test-run/attestation/Dockerfile.mock-attestation b/dstack/gateway/test-run/attestation/Dockerfile.mock-attestation new file mode 100644 index 000000000..416d73304 --- /dev/null +++ b/dstack/gateway/test-run/attestation/Dockerfile.mock-attestation @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# SPDX-License-Identifier: Apache-2.0 + +FROM rust:1.92-bookworm AS builder +WORKDIR /src + +# The toolchain file first, on a layer of its own. +# +# The workspace pins a channel plus rustfmt, clippy, rust-analyzer and three +# extra targets, none of which the base image carries and none of which this +# build needs -- so the first cargo invocation makes rustup download and install +# a complete toolchain. Measured at ~27s, in each of the two fixture images, on +# every build: it happened inside the step below, which `COPY . .` invalidates +# on any change anywhere in the repo. Keyed on the toolchain file alone it is a +# cache hit until the pin actually moves. +COPY rust-toolchain.toml ./rust-toolchain.toml +RUN cargo --version + +# The cargo caches are BuildKit cache mounts, not image layers, because +# `COPY . .` is the whole repo and busts any layer after it on any source +# change. A mount survives that, and both fixture images name the same three, so +# whichever builds second reuses the first's artifacts. `sharing=locked` +# because cargo does not want two builds in one target directory; they +# serialise rather than corrupt. +# +# The binary is copied out inside the RUN: a cache mount is not part of the +# resulting layer, so a later `COPY --from=builder /src/dstack/target/...` +# would find nothing there. +COPY . . +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,target=/src/dstack/target,sharing=locked \ + cargo build --manifest-path dstack/Cargo.toml --locked --release \ + -p mock-attestation \ + && mkdir -p /out && cp dstack/target/release/dstack-mock-attestation /out/ + +FROM debian:bookworm-slim +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates curl && \ + rm -rf /var/lib/apt/lists/* +COPY --from=builder /out/dstack-mock-attestation /usr/local/bin/ +COPY dstack/gateway/test-run/attestation/tee-simulator.json /etc/dstack/tee-simulator.json +ENTRYPOINT ["/usr/local/bin/dstack-mock-attestation"] diff --git a/dstack/gateway/test-run/attestation/Dockerfile.mock-attestation.dockerignore b/dstack/gateway/test-run/attestation/Dockerfile.mock-attestation.dockerignore new file mode 100644 index 000000000..f53ffe346 --- /dev/null +++ b/dstack/gateway/test-run/attestation/Dockerfile.mock-attestation.dockerignore @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# SPDX-License-Identifier: Apache-2.0 + +.git +**/target +**/node_modules +**/__pycache__ +**/.env +**/.env.* diff --git a/dstack/gateway/test-run/attestation/Dockerfile.simulator b/dstack/gateway/test-run/attestation/Dockerfile.simulator new file mode 100644 index 000000000..15a5b2c5b --- /dev/null +++ b/dstack/gateway/test-run/attestation/Dockerfile.simulator @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# SPDX-License-Identifier: Apache-2.0 + +FROM rust:1.92-bookworm AS builder +WORKDIR /src + +# The toolchain file first, on a layer of its own. +# +# The workspace pins a channel plus rustfmt, clippy, rust-analyzer and three +# extra targets, none of which the base image carries and none of which this +# build needs -- so the first cargo invocation makes rustup download and install +# a complete toolchain. Measured at ~27s, in each of the two fixture images, on +# every build: it happened inside the step below, which `COPY . .` invalidates +# on any change anywhere in the repo. Keyed on the toolchain file alone it is a +# cache hit until the pin actually moves. +COPY rust-toolchain.toml ./rust-toolchain.toml +RUN cargo --version + +# The cargo caches are BuildKit cache mounts, not image layers, because +# `COPY . .` is the whole repo and busts any layer after it on any source +# change. A mount survives that, and both fixture images name the same three, so +# whichever builds second reuses the first's artifacts. `sharing=locked` +# because cargo does not want two builds in one target directory; they +# serialise rather than corrupt. +# +# The binary is copied out inside the RUN: a cache mount is not part of the +# resulting layer, so a later `COPY --from=builder /src/dstack/target/...` +# would find nothing there. +COPY . . +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,target=/src/dstack/target,sharing=locked \ + cargo build --manifest-path dstack/Cargo.toml --locked --release \ + -p dstack-guest-agent-simulator \ + && mkdir -p /out && cp dstack/target/release/dstack-simulator /out/ + +FROM debian:bookworm-slim +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates && \ + rm -rf /var/lib/apt/lists/* +WORKDIR /opt/dstack-simulator +COPY --from=builder /out/dstack-simulator /usr/local/bin/dstack-simulator +COPY sdk/simulator/app-compose.json sdk/simulator/appkeys.json \ + sdk/simulator/sys-config.json sdk/simulator/attestation.bin ./ +COPY dstack/gateway/test-run/attestation/simulator.toml ./simulator.toml +CMD ["dstack-simulator", "--config", "/opt/dstack-simulator/simulator.toml"] diff --git a/dstack/gateway/test-run/attestation/Dockerfile.simulator.dockerignore b/dstack/gateway/test-run/attestation/Dockerfile.simulator.dockerignore new file mode 100644 index 000000000..f53ffe346 --- /dev/null +++ b/dstack/gateway/test-run/attestation/Dockerfile.simulator.dockerignore @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# SPDX-License-Identifier: Apache-2.0 + +.git +**/target +**/node_modules +**/__pycache__ +**/.env +**/.env.* diff --git a/dstack/gateway/test-run/attestation/fixture.yml b/dstack/gateway/test-run/attestation/fixture.yml new file mode 100644 index 000000000..8c15efdaf --- /dev/null +++ b/dstack/gateway/test-run/attestation/fixture.yml @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +# Shared attestation fixture for the gateway test suites. +# +# The guest agent simulator signs its quotes under trust anchors derived from +# `mock_attestation_seed`, and the collateral service reconstructs the matching +# public roots from the same seed. The two are only useful together and only +# while the seed matches, so they live in one file that every suite includes +# rather than being copied per suite, where they would drift apart and leave +# every peer quote failing to verify for no visible reason. +# +# Deliberately network-agnostic: an including file attaches these services to +# its own network. Compose merges that in while keeping what is defined here. + +services: + dstack-simulator: + networks: [attestation] + build: + context: ../../../.. + dockerfile: dstack/gateway/test-run/attestation/Dockerfile.simulator + image: dstack-simulator:gateway-e2e + volumes: + - dstack-socket:/var/run/dstack + healthcheck: + test: ["CMD-SHELL", "test -S /var/run/dstack/dstack.sock"] + interval: 1s + timeout: 1s + retries: 30 + + mock-attestation: + networks: [attestation] + build: + context: ../../../.. + dockerfile: dstack/gateway/test-run/attestation/Dockerfile.mock-attestation + image: dstack-mock-attestation:gateway-e2e + # No container_name on purpose. A fixed one is global, so pinning it here + # would stop two suites -- which all include this file -- from being up at + # the same time. Compose still publishes `mock-attestation` as a network + # alias, so everything addressing it by name keeps working. + command: + - serve + - --listen + - 0.0.0.0:8088 + - --config + - /etc/dstack/tee-simulator.json + - --output + - /roots + volumes: + - attestation-roots:/roots + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8088/tpm/aia/root.pem"] + interval: 1s + timeout: 2s + retries: 30 + +# Named explicitly, and namespaced per suite. +# +# A per-test project has to reference these from outside, and compose's default +# `<project>_<name>` would make that a guess about which project happened to +# create them first -- so the names are pinned. `FIXTURE_NS` then keeps each +# suite's copy to itself. +# +# One instance shared by all three suites was the earlier arrangement, on the +# reasoning that the seed signing the quotes and the seed deriving the verifying +# roots must not drift apart. They cannot: both come from files in this +# directory, so three instances started from them agree by construction. What +# sharing the *running* instance bought instead was a teardown race -- whichever +# suite finished first tore down a fixture the others were still using -- and, +# in CI, three workflows that could not run at the same time. Neither is worth +# paying for something the files already guarantee. +networks: + attestation: + name: ${FIXTURE_NS:?FIXTURE_NS must be set} + +volumes: + dstack-socket: + name: ${FIXTURE_NS:?FIXTURE_NS must be set}-socket + attestation-roots: + name: ${FIXTURE_NS:?FIXTURE_NS must be set}-roots diff --git a/dstack/gateway/test-run/attestation/simulator.toml b/dstack/gateway/test-run/attestation/simulator.toml new file mode 100644 index 000000000..3f34c174e --- /dev/null +++ b/dstack/gateway/test-run/attestation/simulator.toml @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# SPDX-License-Identifier: Apache-2.0 + +[default] +workers = 8 +max_blocking = 64 +ident = "dstack Gateway E2E Simulator" +temp_dir = "/tmp" +keep_alive = 10 +log_level = "info" + +[default.core] +keys_file = "/opt/dstack-simulator/appkeys.json" +compose_file = "/opt/dstack-simulator/app-compose.json" +sys_config_file = "/opt/dstack-simulator/sys-config.json" +data_disks = ["/"] + +[default.core.simulator] +attestation_file = "/opt/dstack-simulator/attestation.bin" +patch_report_data = true +# Re-sign the fixture quote under trust anchors derived from this seed instead +# of shipping the fixture's original, now-stale signature. The mock collateral +# service derives the matching roots from the same seed, so the gateways verify +# each other's quotes through the normal production path rather than skipping +# the check. Must stay in sync with the `mock_attestation_seed` in +# `attestation/tee-simulator.json`, which the collateral service reads. Two +# files, one value: the simulator config is TOML read by the simulator and the +# collateral config is JSON read by dstack-mock-attestation, so there is nowhere +# to put it once. They are kept adjacent, and the e2e run fails closed if they +# drift -- every peer quote stops verifying. +mock_attestation_seed = "4747474747474747474747474747474747474747474747474747474747474747" + +[internal] +address = "unix:/var/run/dstack/dstack.sock" +reuse = true diff --git a/dstack/gateway/test-run/attestation/tee-simulator.json b/dstack/gateway/test-run/attestation/tee-simulator.json new file mode 100644 index 000000000..190eceab3 --- /dev/null +++ b/dstack/gateway/test-run/attestation/tee-simulator.json @@ -0,0 +1,5 @@ +{ + "platform": "dstack-tdx", + "mock_attestation_seed": "4747474747474747474747474747474747474747474747474747474747474747", + "collateral_base_url": "http://mock-attestation:8088" +} diff --git a/dstack/gateway/test-run/attestation/tee-simulator.json.license b/dstack/gateway/test-run/attestation/tee-simulator.json.license new file mode 100644 index 000000000..7ed638e2a --- /dev/null +++ b/dstack/gateway/test-run/attestation/tee-simulator.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> + +SPDX-License-Identifier: Apache-2.0 diff --git a/dstack/gateway/test-run/build-gateway-image.sh b/dstack/gateway/test-run/build-gateway-image.sh new file mode 100755 index 000000000..70cacaddc --- /dev/null +++ b/dstack/gateway/test-run/build-gateway-image.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +# Build the gateway as a static musl binary and wrap it in the alpine runtime +# image both test suites run. Shared so the two suites cannot end up testing +# different builds. +# +# Usage: build-gateway-image.sh <dest-dir> [--skip-build] +# <dest-dir> where to place the binary; also the docker build context + +set -e + +TEST_RUN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DSTACK="$(cd "$TEST_RUN_DIR/../.." && pwd)" + +DEST_DIR="${1:?usage: build-gateway-image.sh <dest-dir> [--skip-build]}" +SKIP_BUILD="${2:-}" + +if [ "$SKIP_BUILD" != "--skip-build" ]; then + echo "[INFO] building dstack-gateway (musl static)..." >&2 + (cd "$REPO_DSTACK" && cargo build --release -p dstack-gateway \ + --target x86_64-unknown-linux-musl) + cp "$REPO_DSTACK/target/x86_64-unknown-linux-musl/release/dstack-gateway" "$DEST_DIR/" +fi + +if [ ! -f "$DEST_DIR/dstack-gateway" ]; then + echo "[ERROR] $DEST_DIR/dstack-gateway is missing; run without --skip-build" >&2 + exit 1 +fi + +echo "[INFO] building the gateway image..." >&2 +docker build -t dstack-gateway:test -f "$TEST_RUN_DIR/Dockerfile.gateway" "$DEST_DIR" diff --git a/dstack/gateway/test-run/cluster.sh b/dstack/gateway/test-run/cluster.sh deleted file mode 100755 index 23521bd1d..000000000 --- a/dstack/gateway/test-run/cluster.sh +++ /dev/null @@ -1,442 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> -# -# SPDX-License-Identifier: Apache-2.0 - -# Gateway cluster management script for manual testing - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$SCRIPT_DIR" - -GATEWAY_BIN="${SCRIPT_DIR}/../../target/release/dstack-gateway" -RUN_DIR="run" -CERTS_DIR="$RUN_DIR/certs" -CA_CERT="$CERTS_DIR/gateway-ca.cert" -LOG_DIR="$RUN_DIR/logs" -TMUX_SESSION="gateway-cluster" - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } -log_error() { echo -e "${RED}[ERROR]${NC} $1"; } - -show_help() { - echo "Gateway Cluster Management Script" - echo "" - echo "Usage: $0 <command>" - echo "" - echo "Commands:" - echo " start Start a 3-node gateway cluster in tmux" - echo " stop Stop the cluster (keep tmux session)" - echo " reg Register a random instance" - echo " status Show cluster status" - echo " clean Destroy cluster and clean all data" - echo " attach Attach to tmux session" - echo " help Show this help" - echo "" -} - -# Generate certificates -generate_certs() { - mkdir -p "$CERTS_DIR" - mkdir -p "$RUN_DIR/certbot/live" - - # Generate CA certificate - if [[ ! -f "$CERTS_DIR/gateway-ca.key" ]]; then - log_info "Creating CA certificate..." - openssl genrsa -out "$CERTS_DIR/gateway-ca.key" 2048 2>/dev/null - openssl req -x509 -new -nodes \ - -key "$CERTS_DIR/gateway-ca.key" \ - -sha256 -days 365 \ - -out "$CERTS_DIR/gateway-ca.cert" \ - -subj "/CN=Test CA/O=Gateway Test" \ - 2>/dev/null - fi - - # Generate RPC certificate signed by CA - if [[ ! -f "$CERTS_DIR/gateway-rpc.key" ]]; then - log_info "Creating RPC certificate..." - openssl genrsa -out "$CERTS_DIR/gateway-rpc.key" 2048 2>/dev/null - openssl req -new \ - -key "$CERTS_DIR/gateway-rpc.key" \ - -out "$CERTS_DIR/gateway-rpc.csr" \ - -subj "/CN=localhost" \ - 2>/dev/null - cat > "$CERTS_DIR/ext.cnf" << EXTEOF -authorityKeyIdentifier=keyid,issuer -basicConstraints=CA:FALSE -keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment -subjectAltName = @alt_names - -[alt_names] -DNS.1 = localhost -IP.1 = 127.0.0.1 -EXTEOF - openssl x509 -req \ - -in "$CERTS_DIR/gateway-rpc.csr" \ - -CA "$CERTS_DIR/gateway-ca.cert" \ - -CAkey "$CERTS_DIR/gateway-ca.key" \ - -CAcreateserial \ - -out "$CERTS_DIR/gateway-rpc.cert" \ - -days 365 \ - -sha256 \ - -extfile "$CERTS_DIR/ext.cnf" \ - 2>/dev/null - rm -f "$CERTS_DIR/gateway-rpc.csr" "$CERTS_DIR/ext.cnf" - fi - - # Generate proxy certificates - local proxy_cert_dir="$RUN_DIR/certbot/live" - if [[ ! -f "$proxy_cert_dir/cert.pem" ]]; then - log_info "Creating proxy certificates..." - openssl req -x509 -newkey rsa:2048 -nodes \ - -keyout "$proxy_cert_dir/key.pem" \ - -out "$proxy_cert_dir/cert.pem" \ - -days 365 \ - -subj "/CN=localhost" \ - 2>/dev/null - fi - - # Generate unique WireGuard key pair for each node - for i in 1 2 3; do - if [[ ! -f "$CERTS_DIR/wg-node${i}.key" ]]; then - log_info "Generating WireGuard keys for node ${i}..." - wg genkey > "$CERTS_DIR/wg-node${i}.key" - wg pubkey < "$CERTS_DIR/wg-node${i}.key" > "$CERTS_DIR/wg-node${i}.pub" - fi - done -} - -# Generate node config -generate_config() { - local node_id=$1 - local rpc_port=$((13000 + node_id * 10 + 2)) - local wg_port=$((13000 + node_id * 10 + 3)) - local proxy_port=$((13000 + node_id * 10 + 4)) - local debug_port=$((13000 + node_id * 10 + 5)) - local admin_port=$((13000 + node_id * 10 + 6)) - local wg_ip="10.0.3${node_id}.1/24" - local other_nodes="" - local peer_urls="" - - # Read WireGuard keys for this node - local wg_private_key=$(cat "$CERTS_DIR/wg-node${node_id}.key") - local wg_public_key=$(cat "$CERTS_DIR/wg-node${node_id}.pub") - - for i in 1 2 3; do - if [[ $i -ne $node_id ]]; then - local peer_rpc_port=$((13000 + i * 10 + 2)) - if [[ -n "$other_nodes" ]]; then - other_nodes="$other_nodes, $i" - peer_urls="$peer_urls, \"$i:https://localhost:$peer_rpc_port\"" - else - other_nodes="$i" - peer_urls="\"$i:https://localhost:$peer_rpc_port\"" - fi - fi - done - - local abs_run_dir="$SCRIPT_DIR/$RUN_DIR" - cat > "$RUN_DIR/node${node_id}.toml" << EOF -log_level = "info" -address = "0.0.0.0" -port = ${rpc_port} - -[tls] -key = "${abs_run_dir}/certs/gateway-rpc.key" -certs = "${abs_run_dir}/certs/gateway-rpc.cert" - -[tls.mutual] -ca_certs = "${abs_run_dir}/certs/gateway-ca.cert" -mandatory = false - -[core] -kms_url = "" -rpc_domain = "gateway.test.local" - -[core.debug] -insecure_enable_debug_rpc = true -insecure_skip_attestation = true -port = ${debug_port} -address = "127.0.0.1" - -[core.admin] -enabled = true -port = ${admin_port} -address = "127.0.0.1" - -[core.sync] -enabled = true -interval = "5s" -timeout = "10s" -my_url = "https://localhost:${rpc_port}" -bootnode = "" -node_id = ${node_id} -data_dir = "${RUN_DIR}/wavekv_node${node_id}" - -[core.certbot] -enabled = false - -[core.wg] -private_key = "${wg_private_key}" -public_key = "${wg_public_key}" -listen_port = ${wg_port} -ip = "${wg_ip}" -reserved_net = ["10.0.3${node_id}.1/31"] -client_ip_range = "10.0.3${node_id}.1/24" -config_path = "${RUN_DIR}/wg_node${node_id}.conf" -interface = "gw-test${node_id}" -endpoint = "127.0.0.1:${wg_port}" - -[core.proxy] -cert_chain = "${RUN_DIR}/certbot/live/cert.pem" -cert_key = "${RUN_DIR}/certbot/live/key.pem" -base_domain = "test.local" -listen_addr = "0.0.0.0" -listen_port = ${proxy_port} -tappd_port = 8090 -external_port = ${proxy_port} - -[core.recycle] -enabled = true -interval = "30s" -timeout = "120s" -node_timeout = "300s" -EOF -} - -# Build gateway binary -build_gateway() { - if [[ ! -f "$GATEWAY_BIN" ]]; then - log_info "Building gateway..." - (cd "$SCRIPT_DIR/.." && cargo build --release) - fi -} - -# Start cluster -cmd_start() { - build_gateway - generate_certs - - # Check if tmux session exists - if tmux has-session -t "$TMUX_SESSION" 2>/dev/null; then - log_warn "Cluster already running. Use 'clean' to restart." - cmd_status - return 0 - fi - - log_info "Generating configs..." - mkdir -p "$RUN_DIR" "$LOG_DIR" - for i in 1 2 3; do - generate_config $i - mkdir -p "$RUN_DIR/wavekv_node${i}" - done - - log_info "Starting cluster in tmux session '$TMUX_SESSION'..." - - # Create wrapper scripts that keep running even if gateway exits - for i in 1 2 3; do - cat > "$RUN_DIR/run_node${i}.sh" << RUNEOF -#!/bin/bash -cd "$SCRIPT_DIR" -while true; do - echo "Starting node ${i}..." - sudo RUST_LOG=info $GATEWAY_BIN -c $RUN_DIR/node${i}.toml 2>&1 | tee -a $LOG_DIR/node${i}.log - echo "Node ${i} exited. Press Ctrl+C to stop, or wait 3s to restart..." - sleep 3 -done -RUNEOF - chmod +x "$RUN_DIR/run_node${i}.sh" - done - - # Create tmux session - tmux new-session -d -s "$TMUX_SESSION" -n "node1" - tmux send-keys -t "$TMUX_SESSION:node1" "$RUN_DIR/run_node1.sh" Enter - - sleep 1 - - # Add windows for other nodes - tmux new-window -t "$TMUX_SESSION" -n "node2" - tmux send-keys -t "$TMUX_SESSION:node2" "$RUN_DIR/run_node2.sh" Enter - - tmux new-window -t "$TMUX_SESSION" -n "node3" - tmux send-keys -t "$TMUX_SESSION:node3" "$RUN_DIR/run_node3.sh" Enter - - # Add a shell window - tmux new-window -t "$TMUX_SESSION" -n "shell" - - sleep 3 - - log_info "Cluster started!" - echo "" - cmd_status - echo "" - log_info "Use '$0 attach' to view logs" -} - -# Stop cluster -cmd_stop() { - log_info "Stopping cluster..." - sudo pkill -9 -f "dstack-gateway.*node[123].toml" 2>/dev/null || true - sudo ip link delete gw-test1 2>/dev/null || true - sudo ip link delete gw-test2 2>/dev/null || true - sudo ip link delete gw-test3 2>/dev/null || true - log_info "Cluster stopped" -} - -# Clean everything -cmd_clean() { - cmd_stop - - # Kill tmux session - tmux kill-session -t "$TMUX_SESSION" 2>/dev/null || true - - log_info "Cleaning data..." - sudo rm -rf "$RUN_DIR/wavekv_node"* - sudo rm -f "$RUN_DIR/gateway-state-node"*.json - rm -f "$RUN_DIR/wg_node"*.conf - rm -f "$RUN_DIR/node"*.toml - rm -f "$RUN_DIR/run_node"*.sh - rm -rf "$LOG_DIR" - - log_info "Cleaned" -} - -# Show status -cmd_status() { - echo -e "${BLUE}=== Gateway Cluster Status ===${NC}" - echo "" - - for i in 1 2 3; do - local rpc_port=$((13000 + i * 10 + 2)) - local proxy_port=$((13000 + i * 10 + 4)) - local debug_port=$((13000 + i * 10 + 5)) - local admin_port=$((13000 + i * 10 + 6)) - - if pgrep -f "dstack-gateway.*node${i}.toml" > /dev/null 2>&1; then - echo -e "Node $i: ${GREEN}RUNNING${NC}" - else - echo -e "Node $i: ${RED}STOPPED${NC}" - fi - echo " RPC: https://localhost:${rpc_port}" - echo " Proxy: https://localhost:${proxy_port}" - echo " Debug: http://localhost:${debug_port}" - echo " Admin: http://localhost:${admin_port}" - echo "" - done - - # Show instance count from first running node - for i in 1 2 3; do - local debug_port=$((13000 + i * 10 + 5)) - if pgrep -f "dstack-gateway.*node${i}.toml" > /dev/null 2>&1; then - local response=$(curl -s -X POST "http://localhost:${debug_port}/prpc/GetSyncData" \ - -H "Content-Type: application/json" -d '{}' 2>/dev/null) - if [[ -n "$response" ]]; then - local n_instances=$(echo "$response" | python3 -c "import sys,json; print(len(json.load(sys.stdin).get('instances', [])))" 2>/dev/null || echo "?") - local n_nodes=$(echo "$response" | python3 -c "import sys,json; print(len(json.load(sys.stdin).get('nodes', [])))" 2>/dev/null || echo "?") - echo -e "${BLUE}Cluster State:${NC}" - echo " Nodes: $n_nodes" - echo " Instances: $n_instances" - fi - break - fi - done -} - -# Register a random instance -cmd_reg() { - # Find a running node - local debug_port="" - for i in 1 2 3; do - local port=$((13000 + i * 10 + 5)) - if pgrep -f "dstack-gateway.*node${i}.toml" > /dev/null 2>&1; then - debug_port=$port - break - fi - done - - if [[ -z "$debug_port" ]]; then - log_error "No running nodes found. Start cluster first." - exit 1 - fi - - # Generate random WireGuard key pair - local private_key=$(wg genkey) - local public_key=$(echo "$private_key" | wg pubkey) - - # Generate random IDs - local app_id="app-$(openssl rand -hex 4)" - local instance_id="inst-$(openssl rand -hex 4)" - - log_info "Registering instance..." - log_info " App ID: $app_id" - log_info " Instance ID: $instance_id" - log_info " Public Key: $public_key" - - local response=$(curl -s \ - -X POST "http://localhost:${debug_port}/prpc/RegisterCvm" \ - -H "Content-Type: application/json" \ - -d "{\"client_public_key\": \"$public_key\", \"app_id\": \"$app_id\", \"instance_id\": \"$instance_id\"}" 2>/dev/null) - - if echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); assert 'wg' in d" 2>/dev/null; then - local client_ip=$(echo "$response" | python3 -c "import sys,json; print(json.load(sys.stdin)['wg']['client_ip'])" 2>/dev/null) - log_info "Registered successfully!" - echo -e " Client IP: ${GREEN}$client_ip${NC}" - echo "" - echo "Instance details:" - echo "$response" | python3 -m json.tool 2>/dev/null || echo "$response" - else - log_error "Registration failed:" - echo "$response" | python3 -m json.tool 2>/dev/null || echo "$response" - exit 1 - fi -} - -# Attach to tmux -cmd_attach() { - if tmux has-session -t "$TMUX_SESSION" 2>/dev/null; then - tmux attach -t "$TMUX_SESSION" - else - log_error "No cluster running" - exit 1 - fi -} - -# Main -case "${1:-help}" in - start) - cmd_start - ;; - stop) - cmd_stop - ;; - clean) - cmd_clean - ;; - status) - cmd_status - ;; - reg) - cmd_reg - ;; - attach) - cmd_attach - ;; - help|--help|-h) - show_help - ;; - *) - log_error "Unknown command: $1" - show_help - exit 1 - ;; -esac diff --git a/dstack/gateway/test-run/cluster/docker-compose.yml b/dstack/gateway/test-run/cluster/docker-compose.yml new file mode 100644 index 000000000..29d91ab1d --- /dev/null +++ b/dstack/gateway/test-run/cluster/docker-compose.yml @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +# WaveKV / cluster integration suite. +# +# Driven from the host: the driver rewrites a node's config and stops or starts +# its container between tests, which is what the ported tests need and what a +# runner inside the network could not do without the docker socket. +# +# Each node's config and data directory are bind mounts, not tmpfs: the tests +# read and write node state directly -- 17 of 28 do -- and four perform surgery +# on a stopped node's data directory (wipe it but keep node_uuid, stat the WAL). +# +# Host ports are left for docker to assign and the driver looks them up with +# `docker compose port`. Fixed host ports are a standing collision with whatever +# else the developer is running -- the suite this replaces squatted on 13012-13036 +# and carried a wait-for-the-port-to-free loop because of it. +# +# Config and data live under the test's own name, so a test cannot read or +# write what another one left. CURRENT_TEST is exported by the driver before it +# brings a project up; the default keeps `docker compose` usable by hand. +# +# The config is mounted as a DIRECTORY rather than a file. The driver rewrites +# node configs between tests, and a single-file bind mount pins the original +# inode, so a rewritten file would never be seen inside the container. + +# The fixture is a long-lived project of its own, started once by the driver. +# Each test then gets a throwaway project that joins the fixture's network, so a +# test's containers, logs and data start empty because they are new rather than +# because something cleaned them. +networks: + attestation: + external: true + name: ${FIXTURE_NS:?FIXTURE_NS must be set} + +volumes: + dstack-socket: + external: true + name: ${FIXTURE_NS:?FIXTURE_NS must be set}-socket + attestation-roots: + external: true + name: ${FIXTURE_NS:?FIXTURE_NS must be set}-roots + +services: + gateway-1: &node + image: ${GATEWAY_IMAGE:-dstack-gateway:test} + networks: + attestation: + ports: + - "9012" + - "9015" + - "9016" + volumes: + - ./run/configs/${CURRENT_TEST:-suite}/node1:/etc/gateway + - ./run/data/${CURRENT_TEST:-suite}/node1:/var/lib/gateway + - attestation-roots:/var/lib/attestation:ro + - dstack-socket:/var/run/dstack + environment: + - RUST_LOG=info,dstack_gateway=debug + - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock + cap_add: + - NET_ADMIN + + gateway-2: + <<: *node + ports: + - "9012" + - "9015" + - "9016" + volumes: + - ./run/configs/${CURRENT_TEST:-suite}/node2:/etc/gateway + - ./run/data/${CURRENT_TEST:-suite}/node2:/var/lib/gateway + - attestation-roots:/var/lib/attestation:ro + - dstack-socket:/var/run/dstack + + gateway-3: + <<: *node + ports: + - "9012" + - "9015" + - "9016" + volumes: + - ./run/configs/${CURRENT_TEST:-suite}/node3:/etc/gateway + - ./run/data/${CURRENT_TEST:-suite}/node3:/var/lib/gateway + - attestation-roots:/var/lib/attestation:ro + - dstack-socket:/var/run/dstack diff --git a/dstack/gateway/test-run/cluster/lib.sh b/dstack/gateway/test-run/cluster/lib.sh new file mode 100644 index 000000000..320cd2e8b --- /dev/null +++ b/dstack/gateway/test-run/cluster/lib.sh @@ -0,0 +1,350 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash + +# Host-side driver primitives for the gateway cluster suite. +# +# The tests need to stop a node, rewrite its config, wipe or doctor its data +# directory and start it again. All of that happens here, on the host, against +# bind mounts and `docker compose`. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUN_DIR="$SCRIPT_DIR/run" +CONFIG_DIR="$RUN_DIR/configs" +DATA_DIR="$RUN_DIR/data" +LOG_DIR="$RUN_DIR/logs" + +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m' +log_info() { printf "${BLUE}[INFO]${NC} %s\n" "$1" >&2; } +log_warn() { printf "${YELLOW}[WARN]${NC} %s\n" "$1" >&2; } +log_error() { printf "${RED}[ERROR]${NC} %s\n" "$1" >&2; } +log_success() { printf "${GREEN}[PASS]${NC} %s\n" "$1" >&2; } +log_fail() { printf "${RED}[FAIL]${NC} %s\n" "$1" >&2; } + +# Host-side ports are assigned by docker, not fixed, so the suite cannot +# collide with whatever else is listening on the machine. Look them up once a +# container is running and cache the answer -- `docker compose port` is a +# process spawn, and the tests ask for these constantly. +declare -A _PORT_CACHE=() +container_port() { + local node_id=$1 + local container_port=$2 + # Keyed by project too: each test gets its own containers, so a port + # cached under a previous test's project points at something that no longer + # exists -- which reads as "the node did not come up". + local key="${CURRENT_TEST:-suite}:${node_id}:${container_port}" + local mapped + if [[ -n "${_PORT_CACHE[$key]:-}" ]]; then + echo "${_PORT_CACHE[$key]}"; return 0 + fi + mapped=$(compose port "gateway-${node_id}" "$container_port" 2>/dev/null | awk -F: 'NF{print $NF}') + [[ -n "$mapped" ]] || return 1 + _PORT_CACHE[$key]="$mapped" + echo "$mapped" +} + +# A restarted container gets a new host port, so anything that stops a node has +# to drop the cache or later lookups address a port nothing is listening on. +forget_ports() { + local node_id=$1 + local t="${CURRENT_TEST:-suite}" + unset '_PORT_CACHE['"$t:$node_id"':9012]' \ + '_PORT_CACHE['"$t:$node_id"':9015]' \ + '_PORT_CACHE['"$t:$node_id"':9016]' +} + +rpc_port() { container_port "$1" 9012; } +debug_port() { container_port "$1" 9015; } +admin_port() { container_port "$1" 9016; } + +# Write node $1's config, wiring its bootnode to $2 (empty for none). +# +# Everything except the bootnode is derived from the node id, which is why the +# original suite could regenerate this 30-odd times and only ever change one +# field. +generate_config() { + local node_id=$1 + local bootnode_url=${2:-""} + local dir="$CONFIG_DIR/${CURRENT_TEST:-suite}/node${node_id}" + mkdir -p "$dir" + cat >"$dir/gateway.toml" <<EOF +log_level = "info" +address = "0.0.0.0:9012" + +[tls] +key = "/var/lib/gateway/certs/gateway-rpc.key" +certs = "/var/lib/gateway/certs/gateway-rpc.cert" + +[tls.mutual] +ca_certs = "/var/lib/gateway/certs/gateway-ca.cert" +mandatory = false + +[core] +# Non-empty so the gateway provisions its RPC certificate from the guest agent +# instead of being handed a hand-rolled one. That certificate carries the quote +# and the app_id the cluster's mTLS checks, which is what lets this suite run +# with attestation on rather than switched off. +rpc_domain = "gateway-${node_id}" + +[core.debug] +# TEST ONLY - do not use in production; enables debug RPC +insecure_enable_debug_rpc = true +address = "0.0.0.0:9015" + +[core.admin] +enabled = true +address = "0.0.0.0:9016" +# TEST ONLY - do not use in production; disables admin API authentication +insecure_no_auth = true + +# Verify peer quotes against the development roots the mock collateral service +# derives from the simulator's seed. Verification runs the normal production +# path; this only says the anchor may come from outside the vendor set. +[core.attestation] +insecure_allow_external_trust_anchors = true + +[core.attestation.urls] +pccs = "http://mock-attestation:8088" + +[core.attestation.root_ca] +tdx = "/var/lib/attestation/tdx-root-ca.pem" + +[core.sync] +enabled = true +interval = "5s" +timeout = "10s" +my_url = "https://gateway-${node_id}:9012" +bootnode = "${bootnode_url}" +node_id = ${node_id} +data_dir = "/var/lib/gateway/wavekv" +persist_interval = "5s" + +[core.certbot] +enabled = false + +[core.wg] +private_key = "SEcoI37oGWynhukxXo5Mi8/8zZBU6abg6T1TOJRMj1Y=" +public_key = "xc+7qkdeNFfl4g4xirGGGXHMc0cABuE5IHaLeCASVWM=" +listen_port = 9013 +ip = "10.0.3${node_id}.1/24" +reserved_net = ["10.0.3${node_id}.1/31"] +client_ip_range = "10.0.3${node_id}.1/24" +config_path = "/var/lib/gateway/wg.conf" +interface = "wg-cluster${node_id}" +endpoint = "gateway-${node_id}:9013" + +[core.proxy] +cert_chain = "/etc/gateway/proxy.cert" +cert_key = "/etc/gateway/proxy.key" +base_domain = "cluster.test" +listen_addr = "0.0.0.0" +listen_port = 9014 +tappd_port = 8090 +external_port = 9014 +EOF + # The proxy listener wants its certificate beside the config, and each test + # has a config directory of its own now -- so generating it separately, once, + # left every test after the first with a gateway that exits at startup with + # "failed to read proxy cert_chain". + generate_proxy_cert "$node_id" + log_info "wrote node${node_id} config (bootnode=${bootnode_url:-none})" +} + +# The proxy listener wants a certificate even though this suite never drives the +# data path. Self-signed is enough; the RPC certificates that actually gate +# cluster membership come from the guest agent, not from here. +generate_proxy_cert() { + local node_id=$1 + local dir="$CONFIG_DIR/${CURRENT_TEST:-suite}/node${node_id}" + [ -f "$dir/proxy.cert" ] && return 0 + openssl req -x509 -newkey rsa:2048 -nodes -days 2 \ + -keyout "$dir/proxy.key" -out "$dir/proxy.cert" \ + -subj "/CN=cluster.test" \ + -addext "subjectAltName=DNS:cluster.test,DNS:*.cluster.test" 2>/dev/null +} + +# Each test runs in a compose project of its own, so its containers, its logs +# and its data start empty because they are new -- not because something cleaned +# them. CURRENT_TEST is set by the runner before each test. +compose() { + docker compose -p "cluster-${CURRENT_TEST:-suite}" \ + -f "$SCRIPT_DIR/docker-compose.yml" "$@" +} + +# The attestation fixture, namespaced to this suite. +# +# Each suite runs its own copy rather than sharing one project. The seed that +# signs quotes and the seed that derives the verifying roots come from files in +# `attestation/`, so separate instances agree by construction -- while a shared +# instance meant whichever suite finished first tore it down under the others, +# and meant the three CI workflows could not run at the same time. +FIXTURE_NS="dstack-fixture-cluster" +export FIXTURE_NS + +fixture_compose() { + docker compose -p "$FIXTURE_NS" \ + -f "$SCRIPT_DIR/../attestation/fixture.yml" "$@" +} + +# Every project this suite may have created, asked of the daemon rather than +# guessed. +# +# `compose down` with CURRENT_TEST unset addresses `cluster-suite`, which no +# test ever uses -- the runner sets CURRENT_TEST before the first one -- so the +# `down` subcommand tore down nothing and a killed run leaked one project per +# test, containers still attached to the shared attestation network. Label +# lookup finds them whatever they are called and whatever went wrong. +cluster_projects() { + # Matched on the compose file's absolute path, not on a `cluster-*` name + # pattern. The name is this suite's convention, not something the daemon + # guarantees, and a pattern wide enough to catch every per-test project is + # also wide enough to tear down an unrelated `cluster-…` project that + # happens to be on the same machine. `config_files` is set by compose to the + # file the container came from, so this selects exactly this checkout's + # containers -- another worktree running the same suite is left alone. + docker ps -a \ + --filter "label=com.docker.compose.project.config_files=$SCRIPT_DIR/docker-compose.yml" \ + --format '{{.Label "com.docker.compose.project"}}' 2>/dev/null \ + | sort -u +} + +down_all() { + local project + while read -r project; do + [ -n "$project" ] || continue + docker compose -p "$project" -f "$SCRIPT_DIR/docker-compose.yml" \ + down -v --remove-orphans >/dev/null 2>&1 || true + done < <(cluster_projects) +} + +# Clear the run tree's contents, from inside a container. +# +# The gateway runs as root and writes its certificates 0600 (see +# `safe_write_with_mode` in gateway/src/main.rs), so the host user cannot delete +# what a previous run left. Node state is a bind mount, which `compose down -v` +# does not touch either -- so without this every test resumed from the previous +# run's store. Test names are fixed, so the stale directory is always the one +# the test is about to use: `test_cross_node_data_sync` found its instance +# already replicated and passed with sync entirely broken, and +# `test_partial_cluster_bootstrap` never had to bootstrap. Only the first run on +# a fresh checkout -- which is what CI does, and what this was verified on -- +# behaved as the comments describe. +# +# Contents only. `run` is what the suite's lock is held on, and replacing the +# directory would detach that lock exactly the way a deleted lock file does. +wipe_run_tree() { + mkdir -p "$RUN_DIR" + docker run --rm -v "$RUN_DIR:/r" alpine:latest \ + find /r -mindepth 1 -delete >/dev/null 2>&1 \ + || log_warn "could not clear $RUN_DIR; a stale store may make assertions vacuous" +} + +# Discard whatever the previous test used and start this one from nothing. +new_test_project() { + compose down -v --remove-orphans >/dev/null 2>&1 || true + _PORT_CACHE=() + CURRENT_TEST="$1" + export CURRENT_TEST + mkdir -p "$DATA_DIR/$CURRENT_TEST" +} + +start_node() { + local node_id=$1 + # `compose start` exits 0 when the service has no container yet -- it prints + # "has no container to start" and succeeds -- so an `||` fallback to + # `compose up -d` never fires. Under one project per test the container + # usually does not exist, which made every node after the first look like it + # failed to come up. `up -d` both creates and starts, and is a no-op for a + # container that is already running. + compose up -d "gateway-${node_id}" >/dev/null 2>&1 || + compose start "gateway-${node_id}" >/dev/null 2>&1 + forget_ports "$node_id" + wait_for_debug "$node_id" 60 || { log_error "node ${node_id} did not come up"; return 1; } +} + +stop_node() { + local node_id=$1 + compose stop -t 5 "gateway-${node_id}" >/dev/null 2>&1 || true + forget_ports "$node_id" +} + +wait_for_debug() { + local node_id=$1 + local timeout=${2:-60} + # Wall clock, not an iteration count: each pass forks `docker compose port` + # and a curl, so counting `sleep 1`s overshot the stated timeout by half + # again. See `wait_until` in rpc.sh for the same fix on the sync waits. + local deadline=$((SECONDS + timeout)) + local port + while [ "$SECONDS" -lt "$deadline" ]; do + # Resolved inside the loop on purpose: docker only reports the mapping + # once the container is running, so looking it up once up front would + # leave every later request aimed at an empty port. + if port=$(debug_port "$node_id") && [ -n "$port" ] && + curl -sf -X POST "http://127.0.0.1:${port}/prpc/Debug.Info" \ + -H 'Content-Type: application/json' -d '{}' >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + return 1 +} + +# Two ported tests assert on what a node logged. Containers keep that in the +# daemon rather than a file, so materialise it on demand and leave the +# assertions themselves unchanged. +# +# No window parameter, and none needed: see the note in the body. +dump_log() { + local node_id=$1 + mkdir -p "$LOG_DIR" + # No time window. This container was created for this test, so everything in + # its log belongs to this test -- and the window the shared-container design + # needed is what made the docker-logs timezone bug possible. + compose logs --no-color "gateway-${node_id}" \ + >"$LOG_DIR/${CURRENT_TEST}-node${node_id}.log" 2>&1 || true + echo "$LOG_DIR/${CURRENT_TEST}-node${node_id}.log" +} + + +node_data_dir() { echo "$DATA_DIR/${CURRENT_TEST:-suite}/node$1"; } + +# The gateway runs as root inside its container, so everything it writes into +# the bind-mounted data directory is root-owned. Reading it from the host is +# fine -- node_uuid and the WAL are world-readable, which is all the surgery +# tests need -- but removing or replacing it is not. Do that in a throwaway +# root container rather than reaching for sudo, which is exactly the dependency +# this port exists to shed. +# +# $1 is a shell snippet run against /data, the parent of the per-node dirs. +data_op() { + docker run --rm -v "$DATA_DIR/${CURRENT_TEST}:/data" alpine:latest sh -c "$1" +} + +# Wipe a stopped node's store but let it keep the identity it already published, +# which is what a node looks like after losing its disk but not its config. +# Fails loudly. The `&&` after the first `cp` short-circuits the wipe if +# node_uuid is not there, and errexit is off inside a test body (they run as the +# condition of an `if`), so an unchecked call let +# `test_bootstrap_after_data_dir_loss` "recover" a store that was never lost. +wipe_data_keeping_uuid() { + local node_id=$1 + data_op "set -e + cp /data/node${node_id}/wavekv/node_uuid /tmp/uuid + rm -rf /data/node${node_id}/wavekv/* /data/node${node_id}/wavekv/.[!.]* 2>/dev/null || true + cp /tmp/uuid /data/node${node_id}/wavekv/node_uuid" \ + || { log_error "could not wipe node ${node_id}'s store; the test below would prove nothing"; return 1; } +} + +# Wipe a stopped node completely, identity included -- a stranger turning up +# with a node id that is already taken. Contents only: the directory itself is a +# bind-mount source, and replacing it swaps the inode the mount was set against. +wipe_data() { + local node_id=$1 + # Same reasoning as `wipe_data_keeping_uuid`: a silently skipped wipe leaves + # the following assertions testing the wrong starting state. The globs are + # allowed to match nothing; the operation as a whole is not allowed to fail. + data_op "rm -rf /data/node${node_id}/..?* /data/node${node_id}/.[!.]* /data/node${node_id}/* 2>/dev/null; exit 0" \ + || { log_error "could not wipe node ${node_id}'s data directory"; return 1; } +} diff --git a/dstack/gateway/test-run/cluster/rpc.sh b/dstack/gateway/test-run/cluster/rpc.sh new file mode 100644 index 000000000..8a870f97b --- /dev/null +++ b/dstack/gateway/test-run/cluster/rpc.sh @@ -0,0 +1,310 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash + +# RPC helpers for the cluster suite. +# +# These take a node id rather than a port. The process-based suite passed ports +# around because they were fixed and computable; here docker assigns them, so +# resolving one is a lookup and every caller would otherwise have to do it. + +debug_call() { + local node_id=$1 + local method=$2 + local body=${3:-'{}'} + local port + port=$(debug_port "$node_id") || return 1 + curl -s -X POST "http://127.0.0.1:${port}/prpc/${method}" \ + -H "Content-Type: application/json" -d "$body" 2>/dev/null +} + +admin_call() { + local node_id=$1 + local method=$2 + local body=${3:-'{}'} + local port + port=$(admin_port "$node_id") || return 1 + curl -s -X POST "http://127.0.0.1:${port}/prpc/${method}" \ + -H "Content-Type: application/json" -d "$body" 2>/dev/null +} + +json_field() { python3 -c "$1" 2>/dev/null; } + +check_debug_service() { + debug_call "$1" Debug.Info \ + | json_field "import sys,json; d=json.load(sys.stdin); assert 'base_domain' in d" +} + +debug_get_sync_data() { debug_call "$1" Debug.GetSyncData; } + +# A WireGuard public key is 32 bytes; derive a distinct one per seed so +# registrations do not collide. +test_public_key() { + python3 -c "import base64; print(base64.b64encode(int($1).to_bytes(32, 'big')).decode())" +} + +debug_register_cvm() { + local node_id=$1 + local public_key=$2 + local app_id=${3:-testapp} + local instance_id=${4:-testinstance} + debug_call "$node_id" RegisterCvm \ + "{\"client_public_key\": \"$public_key\", \"app_id\": \"$app_id\", \"instance_id\": \"$instance_id\"}" +} + +# Prints the allocated client_ip, or fails. +verify_register_response() { + echo "$1" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + if 'error' in d: + print(f'ERROR: {d[\"error\"]}', file=sys.stderr) + sys.exit(1) + assert 'wg' in d, 'missing wg config' + assert 'client_ip' in d['wg'], 'missing client_ip' + print(d['wg']['client_ip']) +except Exception as e: + print(f'ERROR: {e}', file=sys.stderr) + sys.exit(1) +" 2>/dev/null +} + +_sync_data_has() { + local node_id=$1 + local collection=$2 + local peer_node_id=$3 + debug_get_sync_data "$node_id" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + for entry in d.get('$collection', []): + if entry.get('node_id') == $peer_node_id: + sys.exit(0) + sys.exit(1) +except Exception: + sys.exit(1) +" +} + +has_peer_addr() { _sync_data_has "$1" peer_addrs "$2"; } +has_node_info() { _sync_data_has "$1" nodes "$2"; } + +get_n_instances() { + debug_get_sync_data "$1" | python3 -c " +import sys, json +try: + print(len(json.load(sys.stdin).get('instances', []))) +except Exception: + print(0) +" 2>/dev/null +} + +get_n_nodes() { + debug_get_sync_data "$1" | python3 -c " +import sys, json +try: + print(len(json.load(sys.stdin).get('nodes', []))) +except Exception: + print(0) +" 2>/dev/null +} + +# Two different endpoints, and conflating them silently breaks whatever reads +# the wrong one: WaveKvStatus carries the store digests and key counts, Status +# carries node identity and the peer list. +admin_wavekv_status() { admin_call "$1" Admin.WaveKvStatus; } +admin_status() { admin_call "$1" Admin.Status; } + +get_store_digest() { + local node_id=$1 + local store=$2 + admin_wavekv_status "$node_id" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['$store']['digest'])" 2>/dev/null || true +} + +# A node's own uuid, as it reports itself in its node list. +get_node_uuid() { + local node_id=$1 + admin_status "$node_id" | python3 -c " +import sys, json +d = json.load(sys.stdin) +me = d.get('id') +for n in d.get('nodes', []): + if n.get('id') == me: + print(n.get('uuid', '')) + break +" 2>/dev/null || true +} + +admin_set_node_url() { + admin_call "$1" Admin.SetNodeUrl "{\"id\": $2, \"url\": \"$3\"}" +} + +# Peers address each other by container hostname, not by a published port: the +# gateways talk over the compose network, and only the driver goes through the +# host. +# Wait until every node in the set has both a peer address and a node record for +# every other. +# +# `setup_peers` only pushes SetNodeUrl; the nodes then have to reach each other +# and record it, which is what the tests were sleeping a flat 4-20s for. Sleeping +# is both slower and weaker: a blind wait cannot tell "peering formed in 800ms" +# from "peering never formed and the assertion below is about to test something +# else". +# +# Both collections, not just `peer_addrs`. They settle at different times: +# `peer_addrs` is written when SetNodeUrl lands, while `nodes` needs an actual +# sync round to carry the record across. Waiting on the first alone returns +# earlier than the blind settle it replaced did, which is how +# `test_multi_node_sync` -- whose own assertions cover both -- started failing on +# `node_info` while `peer_addr` was already there. `nodes` implies a completed +# round, so it is the honest reading of "the nodes found each other" and it is +# what every caller here is really waiting for. +wait_for_peers() { + local timeout_seconds=$1; shift + wait_until "$timeout_seconds" _all_peers_known "$@" +} + +_all_peers_known() { + local node_ids=("$@") src dst + for src in "${node_ids[@]}"; do + for dst in "${node_ids[@]}"; do + [ "$src" = "$dst" ] && continue + has_peer_addr "$src" "$dst" || return 1 + has_node_info "$src" "$dst" || return 1 + done + done + return 0 +} + +setup_peers() { + local node_ids=("$@") + local src dst + for src in "${node_ids[@]}"; do + for dst in "${node_ids[@]}"; do + [ "$src" = "$dst" ] && continue + admin_set_node_url "$src" "$dst" "https://gateway-${dst}:9012" >/dev/null + done + done +} + +# Wall-clock deadline, not an iteration count. +# +# These loops used to run `$((timeout_seconds * 10))` iterations of "probe, then +# sleep 0.1", on the assumption that an iteration costs 0.1s. Every probe forks +# a curl and a python3, so an iteration costs closer to 0.15s and the real +# window was ~1.5x what the caller asked for. That is not a rounding error here: +# `test_push_fast_path` passes 3 to prove a push arrived *before* the 5s +# periodic sync could have done it anyway, and the true window was ~4.8s -- so a +# completely dead push path was caught by the periodic round and the test still +# went green. Bound on time and the number means what it says. +wait_until() { + local timeout_seconds=$1; shift + local deadline=$((SECONDS + timeout_seconds)) + while :; do + "$@" && return 0 + [ "$SECONDS" -lt "$deadline" ] || return 1 + sleep 0.1 + done +} + +_has_n_instances() { + [ "$(get_n_instances "$1")" -ge "$2" ] 2>/dev/null +} + +wait_for_instances() { + local node_id=$1 + local expected=$2 + local timeout_seconds=$3 + wait_until "$timeout_seconds" _has_n_instances "$node_id" "$expected" +} + +_digests_match() { + local d1 d2 + d1=$(get_store_digest "$2" "$1") + d2=$(get_store_digest "$3" "$1") + [ -n "$d1" ] && [ "$d1" = "$d2" ] +} + +wait_for_digest_match() { + local store=$1 + local node_a=$2 + local node_b=$3 + local timeout_seconds=$4 + wait_until "$timeout_seconds" _digests_match "$store" "$node_a" "$node_b" +} + +get_n_keys() { + admin_wavekv_status "$1" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['persistent']['n_keys'])" 2>/dev/null || echo 0 +} + +# The gateway writes its CA where only root can read it, so copy it out for the +# one test that validates the RPC chain rather than skipping verification. +export_ca_cert() { + local node_id=$1 + local dest="$RUN_DIR/node${node_id}-ca.cert" + data_op "cat /data/node${node_id}/certs/gateway-ca.cert" >"$dest" 2>/dev/null + [ -s "$dest" ] || return 1 + echo "$dest" +} + +get_n_peer_addrs() { + debug_get_sync_data "$1" | python3 -c " +import sys, json +try: + print(len(json.load(sys.stdin).get('peer_addrs', []))) +except Exception: + print(0) +" 2>/dev/null +} + +debug_get_proxy_state() { debug_call "$1" GetProxyState; } + +get_n_proxy_state_instances() { + debug_get_proxy_state "$1" | python3 -c " +import sys, json +try: + print(len(json.load(sys.stdin).get('instances', []))) +except Exception: + print(0) +" 2>/dev/null +} + +admin_set_node_status() { + admin_call "$1" Admin.SetNodeStatus "{\"id\": $2, \"status\": \"$3\"}" +} + +get_peer_url_from_sync() { + local node_id=$1 + local peer_node_id=$2 + debug_get_sync_data "$node_id" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + for pa in d.get('peer_addrs', []): + if pa.get('node_id') == $peer_node_id: + print(pa.get('url', '')) + sys.exit(0) + print('') +except Exception: + print('') +" 2>/dev/null +} + +# Whether a RegisterCvm response offered the CVM a given gateway. +response_lists_gateway() { + echo "$1" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + for gw in d.get('gateways', []): + if gw.get('id') == $2: + sys.exit(0) + sys.exit(1) +except Exception: + sys.exit(1) +" +} diff --git a/dstack/gateway/test-run/cluster/run-cluster-tests.sh b/dstack/gateway/test-run/cluster/run-cluster-tests.sh new file mode 100755 index 000000000..61d79ee3e --- /dev/null +++ b/dstack/gateway/test-run/cluster/run-cluster-tests.sh @@ -0,0 +1,259 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +# Gateway cluster (WaveKV) integration suite, driven from the host. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" +# shellcheck source=rpc.sh +source "$SCRIPT_DIR/rpc.sh" +# shellcheck source=tests.sh +source "$SCRIPT_DIR/tests.sh" + +SKIP_BUILD="" +KEEP_RUNNING=false +ONLY="" +while [[ $# -gt 0 ]]; do + case $1 in + --skip-build) SKIP_BUILD="--skip-build"; shift ;; + --keep-running) KEEP_RUNNING=true; shift ;; + --only) ONLY="$2"; shift 2 ;; + down) + # Every per-test project, not just the one CURRENT_TEST happens to + # name, plus the shared fixture -- otherwise CI's teardown leaves + # the fixture project and its global network and volumes behind on + # any runner that outlives the job. + down_all + fixture_compose down -v --remove-orphans >/dev/null 2>&1 || true + exit 0 ;; + -h|--help) + echo "Usage: $0 [--skip-build] [--keep-running] [--only <test>] | down" + exit 0 ;; + *) log_error "unknown option: $1"; exit 1 ;; + esac +done + +# Two runs of this suite share one compose project, one set of container names +# and one data directory, so a second run stops the containers the first is +# using and wipes the store out from under it. The result does not look like a +# collision -- it looks like flaky cross-node sync, because a node's writes +# vanish mid-test. Refuse to start instead. +# The lock is taken on the run directory itself, not on a file inside it. +# Deleting a lock FILE does not release the lock -- it detaches it: the holder +# keeps its inode while the next run creates a fresh one and flocks that +# successfully. Three runs of this suite overlapped that way, each wiping the +# others' state, and the failures read as flaky cross-node sync rather than as +# a collision. A directory is not something a cleanup step removes by name, and +# `rm -rf run/data run/configs run/logs` leaves it in place. +mkdir -p "$RUN_DIR" +exec 9<"$RUN_DIR" +if ! flock -n 9; then + log_error "another run of this suite is already in progress ($RUN_DIR)" + exit 1 +fi + +# No process-name guard beyond the lock. One was tried and false-positived on +# its own first run: `$(...)` forks a subshell whose argv is identical to the +# script's, so `pgrep -f` counted the check itself as a second instance. The +# directory lock above is the correct primitive -- it is race-free and needs no +# pattern matching. + +# Probed BEFORE the trap is installed, not next to the `up` that follows it. +# +# The fixture is shared with the other two suites and is meant to outlive any +# one of them, so cleanup tears it down only if this run started it. Reading +# that after `trap cleanup EXIT` meant every failure in between -- an image +# build, a wipe, a `docker` that is not there -- ran cleanup with the variable +# unset, took the `:-0` default, concluded it had started the fixture, and +# removed a fixture another suite was using. +FIXTURE_WAS_UP=$(fixture_compose ps -q 2>/dev/null | wc -l) + +cleanup() { + $KEEP_RUNNING && return 0 + # The last test's project is still up; the fixture outlives them all. + compose down -v --remove-orphans 2>/dev/null || true + # Only if this run started it. The fixture is meant to outlive a single + # suite; tearing down one this run found already up is what made the next + # run fail with "network dstack-attestation declared as external, but could + # not be found". + if [ "${FIXTURE_WAS_UP:-0}" -eq 0 ]; then + fixture_compose down -v --remove-orphans 2>/dev/null || true + fi +} +trap cleanup EXIT + +TESTS_PASSED=0 +TESTS_FAILED=0 +check() { + local name="$1"; shift + if "$@"; then + log_success "$name"; TESTS_PASSED=$((TESTS_PASSED + 1)) + else + log_fail "$name"; TESTS_FAILED=$((TESTS_FAILED + 1)) + fi +} + +# ---------------------------------------------------------------- assertions + +# Every node must see all three, on every node. +cluster_has_three_nodes() { + local node_id=$1 port count + port=$(debug_port "$node_id") + count=$(curl -sf -X POST "http://127.0.0.1:${port}/prpc/Debug.GetSyncData" \ + -H 'Content-Type: application/json' -d '{}' 2>/dev/null \ + | python3 -c 'import sys,json; print(len(json.load(sys.stdin).get("nodes",[])))' 2>/dev/null) || return 1 + [ "$count" = "3" ] +} + +# Positive evidence that the cluster's mTLS is real: the certificate a node +# serves carries the app_id extension its peers pin, which only happens when it +# was issued through the guest agent against a verifiable quote. Absence of +# errors would not prove this -- a validator that never ran is also silent. +cert_carries_app_id() { + local node_id=$1 + data_op "cat /data/node${node_id}/certs/gateway-rpc.cert" 2>/dev/null \ + | openssl x509 -noout -text 2>/dev/null \ + | grep -q "1.3.6.1.4.1.62397.1.3" +} + +# And the negative: no peer was ever turned away. +# +# Deliberately not matching "bootnode discovery retry failed". A node whose +# bootnode is not listening yet retries and succeeds, which is a startup race, +# not an authentication failure -- treating it as one made this assertion fail +# on whichever node happened to start first. Convergence is what proves the +# retry worked, and that is asserted separately. +no_peer_was_rejected() { + local node_id=$1 log + log=$(dump_log "$node_id") + # `dump_log` ends in `|| true`, and `! grep -q` on an empty file is true, so + # without this the assertion passed whenever the log could not be collected + # at all -- the failure mode a negative assertion is most exposed to. Prove + # there is something to have searched first. + [ -s "$log" ] || { log_error "node ${node_id} produced no log to check"; return 1; } + ! grep -qE "does not contain app_id|invalid quote|app_id mismatch" "$log" +} + +# ---------------------------------------------------------------- main + +log_info "==========================================" +log_info "dstack-gateway cluster suite" +log_info "==========================================" + +# Discard anything a previous run left: node state is a bind mount that +# `compose down -v` does not remove and the host user cannot delete. +down_all +wipe_run_tree +mkdir -p "$DATA_DIR" + +log_info "starting the attestation fixture" +fixture_compose build >/dev/null +fixture_compose up -d --wait >/dev/null + +"$SCRIPT_DIR/../build-gateway-image.sh" "$SCRIPT_DIR" $SKIP_BUILD + +log_info "building the suite image" +compose build >/dev/null + +# The smoke checks get a project of their own for the same reason every test +# does: the first test's new_test_project tears down whatever came before it, +# and without this the smoke nodes were what it tore down. +# +# It has to come before the configs are written: config and data paths are named +# after CURRENT_TEST, so generating them first puts them where nothing looks. +new_test_project smoke + +log_info "generating node configs" +generate_config 1 "" +generate_config 2 "https://gateway-1:9012" +generate_config 3 "https://gateway-1:9012" + +log_info "starting the cluster" +compose up -d >/dev/null + +for n in 1 2 3; do + check "node $n debug service is up" wait_for_debug "$n" 90 +done + +log_info "waiting for the cluster to converge" +converged=false +for _ in $(seq 1 30); do + if cluster_has_three_nodes 1 && cluster_has_three_nodes 2 && cluster_has_three_nodes 3; then + converged=true; break + fi + sleep 2 +done +$converged || log_warn "cluster did not converge within 60s" + +for n in 1 2 3; do + check "node $n sees all three nodes" cluster_has_three_nodes "$n" + check "node $n serves a certificate carrying its app_id" cert_carries_app_id "$n" + check "node $n rejected no peer" no_peer_was_rejected "$n" +done + +# ------------------------------------------------------------- ported tests +# +# Every row here must also be ticked in the port checklist only once it has been +# seen to FAIL against a broken cluster. Green on its own proves nothing. +run_ported() { + local name="$1" + log_info "---------- $name ----------" + new_test_project "$name" + if "$name"; then + log_success "$name"; TESTS_PASSED=$((TESTS_PASSED + 1)) + else + log_fail "$name"; TESTS_FAILED=$((TESTS_FAILED + 1)) + # Under one project per test the next test's new_test_project tears this + # one down, so a failure's container logs are gone before anyone can + # read them. Dumping them here is what makes a failure diagnosable at + # all: test_partial_cluster_bootstrap failed with nothing in the suite + # log but its own config lines, and there was no way to tell which step + # had failed. An explicit `if` rather than `A && B`, because SC2015 has + # already made a successful run exit non-zero once. + for _n in 1 2 3; do + if compose ps -q "gateway-${_n}" 2>/dev/null | grep -q .; then + log_info " saved log: $(dump_log "$_n")" + fi + done + fi +} + +ALL_TESTS=(test_persistence test_status_endpoint test_prpc_register \ + test_prpc_info test_wal_integrity \ + test_multi_node_sync test_node_recovery test_cross_node_data_sync \ + test_push_fast_path test_periodic_repair_after_missed_push \ + test_bootstrap_after_data_dir_loss test_divergent_partition_writes \ + test_push_periodic_overlap test_delayed_bootnode_recovery \ + test_interrupted_sync_recovery test_ephemeral_recovery \ + test_partial_cluster_bootstrap test_node_id_reuse_rejected \ + test_client_registration_persistence test_stress_writes \ + test_network_partition test_three_node_cluster \ + test_three_node_bootnode test_periodic_persistence \ + test_admin_set_node_url test_admin_set_node_status \ + test_node_status_register_exclude test_node_status_register_reject) + +# An unknown --only name used to select nothing, leave TESTS_FAILED at 0 and +# exit 0: a green run of zero tests, which is the one result this suite must +# never produce. +if [ -n "$ONLY" ]; then + printf '%s\n' "${ALL_TESTS[@]}" | grep -qx -- "$ONLY" || { + log_error "unknown test: $ONLY" + exit 1 + } +fi + +for t in "${ALL_TESTS[@]}"; do + [ -n "$ONLY" ] && [ "$t" != "$ONLY" ] && continue + run_ported "$t" +done + +log_info "==========================================" +log_info "Passed: $TESTS_PASSED" +log_info "Failed: $TESTS_FAILED" +log_info "==========================================" +[ "$TESTS_FAILED" -eq 0 ] diff --git a/dstack/gateway/test-run/cluster/tests.sh b/dstack/gateway/test-run/cluster/tests.sh new file mode 100644 index 000000000..da6c8cbd3 --- /dev/null +++ b/dstack/gateway/test-run/cluster/tests.sh @@ -0,0 +1,903 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash + +# Ported from test-run/test_suite.sh. Each test keeps the assertions it had; +# what changed is how nodes are started, stopped and reset. + +# Nothing to clean: the runner gave this test a compose project of its own, so +# its containers and its data directory are new. Kept as a no-op so each test +# still reads as declaring where it starts from. +cleanup_cluster() { :; } + +# ------------------------------------------------------------------ quick + +test_persistence() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + + sleep 2 + local keys_after_write + keys_after_write=$(get_n_keys 1) + log_info "keys after startup: $keys_after_write" + + stop_node 1 + start_node 1 || return 1 + + local keys_after_restart + keys_after_restart=$(get_n_keys 1) + log_info "keys after restart: $keys_after_restart" + + # get_n_keys ends in `|| echo 0`, so an unreachable node reports zero keys + # rather than failing. Without this guard a node that never wrote anything + # gives 0 >= 0 and the test passes having proved nothing -- the same shape + # as the empty-store digest match that made test_ephemeral_recovery hollow. + [ "${keys_after_write:-0}" -gt 0 ] 2>/dev/null || { + log_error "nothing was in the store before the restart, so surviving it proves nothing: keys=${keys_after_write:-0}" + return 1; } + if [ "$keys_after_restart" -ge "$keys_after_write" ] 2>/dev/null; then + return 0 + fi + log_error "expected >= $keys_after_write keys, got $keys_after_restart" + return 1 +} + +test_status_endpoint() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + + admin_wavekv_status 1 | python3 -c " +import sys, json +d = json.load(sys.stdin) +assert d['enabled'] is True, 'enabled should be True' +assert 'persistent' in d, 'missing persistent' +assert 'ephemeral' in d, 'missing ephemeral' +assert d['persistent']['wal_enabled'] is True, 'persistent wal should be enabled' +assert d['ephemeral']['wal_enabled'] is False, 'ephemeral wal should be disabled' +assert 'peers' in d['persistent'], 'missing peers in persistent' +" +} + +test_prpc_register() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local response + response=$(debug_register_cvm 1 "$(test_public_key 501)" deadbeef cafebabe) + verify_register_response "$response" >/dev/null || { + log_error "RegisterCvm did not return a client_ip"; return 1; } +} + +test_prpc_info() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + + local ca port + ca=$(export_ca_cert 1) || { log_error "could not read the node's CA certificate"; return 1; } + port=$(rpc_port 1) || return 1 + + curl -s --cacert "$ca" --resolve "gateway-1:${port}:127.0.0.1" \ + -X POST "https://gateway-1:${port}/prpc/Info" \ + -H "Content-Type: application/json" -d '{}' 2>/dev/null | python3 -c " +import sys, json +d = json.load(sys.stdin) +if 'error' in d: + print(f'ERROR: {d[\"error\"]}', file=sys.stderr) + sys.exit(1) +assert 'base_domain' in d, 'missing base_domain' +assert 'external_port' in d, 'missing external_port' +" +} + +test_wal_integrity() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local i key response success=0 + for i in $(seq 1 5); do + key=$(test_public_key $((800 + i))) + response=$(debug_register_cvm 1 "$key" "wal_app$i" "wal_inst$i") + verify_register_response "$response" >/dev/null 2>&1 && success=$((success + 1)) + done + [ "$success" -eq 5 ] || { log_error "registered only $success/5 clients"; return 1; } + + # The WAL has to exist and be non-empty: a store that silently stopped + # journalling would still answer every RPC above. + local wal_size + wal_size=$(data_op "stat -c%s /data/node1/wavekv/node_1.wal 2>/dev/null || echo 0" | tr -d '\r') + log_info "WAL size: ${wal_size} bytes" + [ "${wal_size:-0}" -gt 0 ] 2>/dev/null +} + +# ------------------------------------------------------------------- sync + +# Two nodes must learn each other's address and node record, in both directions. +test_multi_node_sync() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 10 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + local ok=0 + has_peer_addr 1 2 || { log_error "node 1 missing peer_addr for node 2"; ok=1; } + has_peer_addr 2 1 || { log_error "node 2 missing peer_addr for node 1"; ok=1; } + has_node_info 1 2 || { log_error "node 1 missing node_info for node 2"; ok=1; } + has_node_info 2 1 || { log_error "node 2 missing node_info for node 1"; ok=1; } + return $ok +} + +# A node that was down has to catch up once it is back. +test_node_recovery() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 5 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + stop_node 2 + sleep 3 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 10 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + local ok=0 + has_peer_addr 2 1 || { log_error "node 2 missing peer_addr for node 1 after recovery"; ok=1; } + has_node_info 2 1 || { log_error "node 2 missing node_info for node 1 after recovery"; ok=1; } + return $ok +} + +# A CVM registered on one node must appear on the other, and each node's two +# views of it -- the replicated store and the proxy's own state -- must agree. +test_cross_node_data_sync() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 5 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + check_debug_service 1 || { log_error "debug service not available on node 1"; return 1; } + + local client_ip + client_ip=$(verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" app1 inst1)") + [ -n "$client_ip" ] || { log_error "registration failed"; return 1; } + + # Both stores on both nodes, then read all four for the equality checks + # below. `|| true` because the assertions are what report -- reaching the + # ceiling should produce the counts that explain it, not a bare timeout. + wait_until 20 _both_nodes_hold_an_instance || true + local kv1 kv2 ps1 ps2 ok=0 + kv1=$(get_n_instances 1); kv2=$(get_n_instances 2) + ps1=$(get_n_proxy_state_instances 1); ps2=$(get_n_proxy_state_instances 2) + + { [ "$kv1" -ge 1 ] && [ "$kv2" -ge 1 ]; } || { + log_error "KvStore sync failed: kv1=$kv1 kv2=$kv2"; ok=1; } + { [ "$ps1" -ge 1 ] && [ "$ps2" -ge 1 ]; } || { + log_error "ProxyState sync failed: ps1=$ps1 ps2=$ps2"; ok=1; } + [ "$kv1" -eq "$ps1" ] || { log_error "node 1 inconsistent: KvStore=$kv1 ProxyState=$ps1"; ok=1; } + [ "$kv2" -eq "$ps2" ] || { log_error "node 2 inconsistent: KvStore=$kv2 ProxyState=$ps2"; ok=1; } + return $ok +} + +_node1_logged_a_periodic_persist() { + grep -q "periodic persist completed" "$(dump_log 1)" +} + +_both_nodes_hold_an_instance() { + local n kv ps + for n in 1 2; do + kv=$(get_n_instances "$n") + ps=$(get_n_proxy_state_instances "$n") + # Same reasoning as `_three_node_views_converged`: the caller asserts + # `kv == ps` as well, so the wait has to cover it. + [ "${kv:-0}" -ge 1 ] 2>/dev/null || return 1 + [ "${ps:-0}" -ge 1 ] 2>/dev/null || return 1 + [ "${kv:-0}" -eq "${ps:-1}" ] 2>/dev/null || return 1 + done + return 0 +} + +# An opportunistic push must land well inside the 5s periodic interval. +test_push_fast_path() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 6 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + local before + before=$(get_n_instances 2) + verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" push_app push_instance)" \ + >/dev/null || return 1 + + # 3s < the 5s periodic interval, so arriving in time is what proves the push + # path ran rather than the anti-entropy round that would have caught it anyway. + wait_for_instances 2 $((before + 1)) 3 || { + log_error "node 2 did not receive the write before the periodic interval"; return 1; } + wait_for_digest_match persistent 1 2 3 || { + log_error "persistent digests did not converge after push"; return 1; } +} + +# A write made while a peer was down must still reach it, via the periodic round. +test_periodic_repair_after_missed_push() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 6 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + local before + before=$(get_n_instances 2) + stop_node 2 + verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" repair_app repair_instance)" \ + >/dev/null || return 1 + sleep 1 + start_node 2 || return 1 + setup_peers 1 2 + + wait_for_instances 2 $((before + 1)) 15 || { + log_error "periodic sync did not repair the missed write"; return 1; } +} + +# Losing the local store must not cost the node its identity, and it must +# refill from the peer rather than come back empty. +test_bootstrap_after_data_dir_loss() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "https://gateway-1:9012" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 6 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" bootstrap_app bootstrap_instance)" \ + >/dev/null || return 1 + wait_for_instances 2 1 10 || return 1 + + local old_uuid new_uuid + old_uuid=$(get_node_uuid 2) + { [ -n "$old_uuid" ] && [ "$old_uuid" != "null" ]; } || { + log_error "node 2 did not report its identity before recovery"; return 1; } + + stop_node 2 + wipe_data_keeping_uuid 2 || return 1 + start_node 2 || return 1 + + wait_for_instances 2 1 15 || { + log_error "node 2 did not bootstrap after losing its local store"; return 1; } + # get_store_digest ends in `|| true`, so an unreachable node yields an empty + # string and empty equals empty. wait_for_instances above speaks for node 2; + # nothing has spoken for node 1, so check both are non-empty before + # concluding anything from their being equal. + local d1 d2 + d1=$(get_store_digest 1 persistent); d2=$(get_store_digest 2 persistent) + { [ -n "$d1" ] && [ -n "$d2" ]; } || { + log_error "could not read both digests, so comparing them proves nothing: node1='${d1}' node2='${d2}'" + return 1; } + [ "$d1" = "$d2" ] || { + log_error "persistent digests differ after bootstrap: node1='${d1}' node2='${d2}'"; return 1; } + + setup_peers 1 2 + wait_for_peers 6 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + new_uuid=$(get_node_uuid 2) + { [ -n "$new_uuid" ] && [ "$new_uuid" != "null" ]; } || { + log_error "node 2 did not report its post-recovery identity"; return 1; } + [ "$old_uuid" = "$new_uuid" ] || { + log_error "losing the WaveKV store unexpectedly changed the node UUID"; return 1; } +} + +# Writes made on both sides of a partition must both survive the merge. +test_divergent_partition_writes() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 6 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + stop_node 2 + verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" left_app left_instance)" \ + >/dev/null || return 1 + + stop_node 1 + start_node 2 || return 1 + verify_register_response \ + "$(debug_register_cvm 2 "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBA=" right_app right_instance)" \ + >/dev/null || return 1 + + start_node 1 || return 1 + setup_peers 1 2 + { wait_for_instances 1 2 15 && wait_for_instances 2 2 15; } || { + log_error "divergent partition writes did not merge"; return 1; } + wait_for_digest_match persistent 1 2 10 || { + log_error "persistent digests did not converge after divergent writes"; return 1; } +} + +# Writes that land while a periodic round is in flight must neither be lost nor +# counted twice. +test_push_periodic_overlap() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 4 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + local before i + before=$(get_n_instances 1) + for i in $(seq 1 6); do + verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key $((100 + i)))" "overlap_app_$i" "overlap_instance_$i")" \ + >/dev/null || { log_error "overlap write $i was rejected"; return 1; } + sleep 0.2 + done + + wait_for_instances 2 $((before + 6)) 15 || { + log_error "writes racing periodic sync did not arrive"; return 1; } + [ "$(get_n_instances 1)" -eq $((before + 6)) ] || { + log_error "overlapping push and sync produced duplicate instances"; return 1; } + wait_for_digest_match persistent 1 2 10 +} + +# A node whose bootnode is not up yet must keep retrying and still converge. +test_delayed_bootnode_recovery() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "https://gateway-1:9012" + start_node 2 || return 1 + verify_register_response \ + "$(debug_register_cvm 2 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" delayed_app delayed_instance)" \ + >/dev/null || return 1 + + sleep 2 + start_node 1 || return 1 + + local _ + for _ in $(seq 1 25); do + has_peer_addr 1 2 && has_peer_addr 2 1 && break + sleep 1 + done + { has_peer_addr 1 2 && has_peer_addr 2 1; } || { + log_error "bootnode retry did not form the cluster"; return 1; } + wait_for_instances 1 1 15 || { + log_error "data did not converge after delayed bootnode recovery"; return 1; } +} + +# Killing a node mid-sync must not wedge it; the next round has to finish the job. +test_interrupted_sync_recovery() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 6 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + stop_node 2 + local i + for i in $(seq 1 20); do + verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key $((200 + i)))" "interrupt_app_$i" "interrupt_instance_$i")" \ + >/dev/null || { log_error "interrupted-sync fixture write $i was rejected"; return 1; } + done + + # Bring it up, let a sync begin, then cut it off again. The exact instant + # does not matter -- what is asserted is that the following round still + # converges, not that the cut landed at a particular byte. + start_node 2 || return 1 + setup_peers 1 2 + sleep 0.2 + stop_node 2 + + start_node 2 || return 1 + setup_peers 1 2 + wait_for_instances 2 20 20 || { + log_error "sync did not recover after interruption"; return 1; } + wait_for_digest_match persistent 1 2 10 +} + +# The ephemeral store is not journalled, so after a restart it has to be rebuilt +# from peers rather than read back from disk. +test_ephemeral_recovery() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 8 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + stop_node 2 + sleep 2 + start_node 2 || return 1 + setup_peers 1 2 + + local keys1 keys2 _ + for _ in $(seq 1 20); do + keys1=$(debug_get_sync_data 1 | python3 -c "import sys,json; print(json.load(sys.stdin)['ephemeral_keys'])" 2>/dev/null || echo 0) + keys2=$(debug_get_sync_data 2 | python3 -c "import sys,json; print(json.load(sys.stdin)['ephemeral_keys'])" 2>/dev/null || echo 0) + { [ "${keys1:-0}" -gt 0 ] && [ "${keys2:-0}" -gt 0 ]; } && break + sleep 1 + done + + # Assert what the loop above only waited for. Matching digests are not + # enough on their own: two nodes that never found each other both hold an + # empty ephemeral store, and empty matches empty. Stubbing out setup_peers + # left this test green, which is how the gap was found. + { [ "${keys1:-0}" -gt 0 ] && [ "${keys2:-0}" -gt 0 ]; } || { + log_error "ephemeral stores are empty, so a digest match proves nothing: keys1=${keys1:-0} keys2=${keys2:-0}" + return 1; } + + wait_for_digest_match ephemeral 1 2 15 || { + log_error "ephemeral store did not converge after restart"; return 1; } +} + +# A joining node must be able to bootstrap from the one peer that is up, even +# though another cluster member is unreachable. +test_partial_cluster_bootstrap() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + generate_config 3 "https://gateway-1:9012" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 6 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" partial_app partial_instance)" \ + >/dev/null || { log_error "node 1 rejected the registration"; return 1; } + # 15s, not 10: every comparable wait in this suite allows 15, and this one + # is a peer sync over a link that setup_peers has only just established. + wait_for_instances 2 1 15 || { + log_error "node 2 never saw the instance registered on node 1"; return 1; } + + stop_node 2 + start_node 3 || return 1 + wait_for_instances 3 1 20 || { + log_error "node 3 did not bootstrap while node 2 was unavailable"; return 1; } +} + +# A node that lost its identity but kept its node id must be rejected on the +# first exchange, must not cost the established peer any data, and must then be +# allowed to rejoin under its new identity rather than staying wedged. +test_node_id_reuse_rejected() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 10 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + has_peer_addr 1 2 || { log_error "node 1 missing peer_addr for node 2"; return 1; } + has_peer_addr 2 1 || { log_error "node 2 missing peer_addr for node 1"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key 400)" reuse_app reuse_fixture)" \ + >/dev/null || return 1 + wait_for_instances 2 1 10 || { log_error "node 2 did not receive the recovery fixture"; return 1; } + + local old_uuid new_uuid keys_before keys_after + old_uuid=$(get_node_uuid 2) + keys_before=$(get_n_keys 1) + + + stop_node 2 + wipe_data 2 || return 1 + start_node 2 || return 1 + + new_uuid=$(get_node_uuid 2) + { [ -n "$old_uuid" ] && [ -n "$new_uuid" ] && [ "$old_uuid" != "$new_uuid" ]; } || { + log_error "fresh node 2 did not receive a new UUID"; return 1; } + + setup_peers 1 2 + + local seen=false log1 log2 _ + for _ in $(seq 1 15); do + log1=$(dump_log 1); log2=$(dump_log 2) + if grep -q "UUID mismatch" "$log1" "$log2" 2>/dev/null; then seen=true; break; fi + sleep 1 + done + [ "$seen" = true ] || { log_error "reused node ID was not rejected"; return 1; } + + # Deliberate deviation from the suite this was ported from, which compared + # total key counts before and after and required the count not to drop. + # + # It does drop, by exactly one, and permanently: node 1 discards node 2's + # superseded identity record, which is the correct thing to do with it. + # Measured 9 -> 8, still 8 thirty seconds later, with the registered + # instance present throughout. So the old assertion forbids a legitimate + # deletion, and passed only when the replacement record happened to land + # before the count was taken. + # + # What the test is actually about -- its own comment says "node 1's data is + # still intact" -- is the replicated instance records, so assert on those. + local instances_after + instances_after=$(get_n_instances 1) + [ "$instances_after" -ge 1 ] 2>/dev/null || { + log_error "node 1 lost its registered instances after node 2 restarted with a reused ID: $instances_after" + return 1; } + log_info "node 1 keys ${keys_before} -> $(get_n_keys 1), instances ${instances_after}" + + wait_for_instances 2 1 20 || { + log_error "fresh node did not recover after the UUID rejection"; return 1; } + # 40s, not the 15 this asked for before. + # + # Recovery here needs the rejected node's fresh identity record to reach + # node 1 in a sync *response* -- its own requests still fail node 1's + # inbound check -- and then an anti-entropy round to carry the store, so it + # costs several 5s intervals rather than one. Measured three times on an + # idle machine: 16s, 17s, 18s. + # + # It passed at 15 only because the wait helpers used to count iterations + # instead of seconds and so ran roughly twice as long as they claimed (see + # `wait_until` in rpc.sh). With the timer made honest this became a coin + # flip -- it passed on one full run and failed on the next. The number now + # says what the operation needs, with headroom for a loaded runner. + wait_for_digest_match persistent 1 2 40 || { + log_error "stores did not converge after UUID recovery"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 2 "$(test_public_key 401)" reuse_app post_recovery)" \ + >/dev/null || return 1 + wait_for_instances 1 2 15 || { log_error "post-recovery write did not propagate"; return 1; } + wait_for_digest_match persistent 1 2 10 +} + +# --------------------------------------------------------------- advanced + +# A registration must survive a restart, and the store must hold more than the +# handful of keys a bare node writes for itself. +test_client_registration_persistence() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local client_ip keys_before keys_after + client_ip=$(verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key 502)" persist_app persist_inst)") + [ -n "$client_ip" ] || { log_error "registration failed"; return 1; } + + keys_before=$(get_n_keys 1) + stop_node 1 + start_node 1 || return 1 + keys_after=$(get_n_keys 1) + + { [ "$keys_after" -ge "$keys_before" ] && [ "$keys_before" -gt 2 ]; } || { + log_error "keys_before=$keys_before keys_after=$keys_after"; return 1; } +} + +test_stress_writes() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local i key app inst success=0 + for i in $(seq 1 10); do + key=$(test_public_key $((600 + i))) + app=$(printf "stressapp%02d" "$i") + inst=$(printf "stressinst%02d" "$i") + verify_register_response "$(debug_register_cvm 1 "$key" "$app" "$inst")" >/dev/null 2>&1 \ + && success=$((success + 1)) + done + sleep 2 + + local keys_after + keys_after=$(get_n_keys 1) + { [ "$success" -eq 10 ] && [ "$keys_after" -gt 2 ]; } || { + log_error "success=$success keys_after=$keys_after"; return 1; } +} + +# Writes accepted while the peer was gone must reach it afterwards, and each +# node's two views must still agree once they do. +test_network_partition() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 5 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + check_debug_service 1 || { log_error "debug service not available on node 1"; return 1; } + + stop_node 2 + local i key success=0 + for i in $(seq 1 3); do + key=$(test_public_key $((700 + i))) + verify_register_response "$(debug_register_cvm 1 "$key" "partition_app$i" "partition_inst$i")" \ + >/dev/null 2>&1 && success=$((success + 1)) + done + + local kv1_during ps1_during + kv1_during=$(get_n_instances 1) + ps1_during=$(get_n_proxy_state_instances 1) + + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 15 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + # Peering is not the thing being waited for here. What the assertions below + # test is that the rejoining node caught up on everything written while it + # was away, which takes an anti-entropy round after the peer records land -- + # the blind settle this replaced covered both, and waiting only for peering + # would return before the catch-up and leave the counts to a race. + wait_for_instances 2 "$kv1_during" 15 || { + log_error "node 2 did not catch up after the partition healed"; return 1; } + + local kv1 kv2 ps1 ps2 ok=0 + kv1=$(get_n_instances 1); kv2=$(get_n_instances 2) + ps1=$(get_n_proxy_state_instances 1); ps2=$(get_n_proxy_state_instances 2) + + { [ "$success" -eq 3 ] && [ "$kv1_during" -ge 3 ]; } || { + log_error "registration or KvStore write failed during partition"; ok=1; } + [ "$kv2" -ge "$kv1_during" ] || { + log_error "node 2 KvStore sync failed: kv2=$kv2 expected >= $kv1_during"; ok=1; } + [ "$ps2" -ge "$kv1_during" ] || { + log_error "node 2 ProxyState sync failed: ps2=$ps2 expected >= $kv1_during"; ok=1; } + [ "$kv1" -eq "$ps1" ] || { log_error "node 1 inconsistent: KvStore=$kv1 ProxyState=$ps1"; ok=1; } + [ "$kv2" -eq "$ps2" ] || { log_error "node 2 inconsistent: KvStore=$kv2 ProxyState=$ps2"; ok=1; } + log_info "ps1_during=$ps1_during" + return $ok +} + +# The same condition as `_three_node_views_agree`, without the logging. +# +# Polling the loud one would print a diagnosis on every failed attempt, so the +# log fills with the states the cluster passed through on its way to the right +# one. Wait on this, then assert with the loud one, which reports the state that +# actually stood. +_three_node_views_converged() { + local n kv ps + for n in 1 2 3; do + kv=$(get_n_instances "$n") + ps=$(get_n_proxy_state_instances "$n") + # Every condition the loud twin asserts, including the equality. Waiting + # on a weaker predicate than the one being asserted is how a wait exits + # early on a state the assertion then rejects -- `kv=2, ps=1` is a real + # intermediate here, because the two are updated by different paths. + [ "${kv:-0}" -ge 1 ] 2>/dev/null || return 1 + [ "${ps:-0}" -ge 1 ] 2>/dev/null || return 1 + [ "${kv:-0}" -eq "${ps:-1}" ] 2>/dev/null || return 1 + done + return 0 +} + +_three_node_views_agree() { + local kv1 kv2 kv3 ps1 ps2 ps3 ok=0 + kv1=$(get_n_instances 1); kv2=$(get_n_instances 2); kv3=$(get_n_instances 3) + ps1=$(get_n_proxy_state_instances 1); ps2=$(get_n_proxy_state_instances 2) + ps3=$(get_n_proxy_state_instances 3) + { [ "$kv1" -ge 1 ] && [ "$kv2" -ge 1 ] && [ "$kv3" -ge 1 ]; } || { + log_error "KvStore sync failed: kv1=$kv1 kv2=$kv2 kv3=$kv3"; ok=1; } + { [ "$ps1" -ge 1 ] && [ "$ps2" -ge 1 ] && [ "$ps3" -ge 1 ]; } || { + log_error "ProxyState sync failed: ps1=$ps1 ps2=$ps2 ps3=$ps3"; ok=1; } + { [ "$kv1" -eq "$ps1" ] && [ "$kv2" -eq "$ps2" ] && [ "$kv3" -eq "$ps3" ]; } || { + log_error "inconsistency between KvStore and ProxyState"; ok=1; } + return $ok +} + +test_three_node_cluster() { + cleanup_cluster + generate_config 1 ""; generate_config 2 ""; generate_config 3 "" + start_node 1 || return 1 + start_node 2 || return 1 + start_node 3 || return 1 + setup_peers 1 2 3 + wait_for_peers 10 1 2 3 || { + log_error "nodes 1 2 3 did not learn about each other"; return 1; } + check_debug_service 1 || { log_error "debug service not available on node 1"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key 503)" threenode_app threenode_inst)" \ + >/dev/null || { log_error "registration failed"; return 1; } + wait_until 20 _three_node_views_converged || true + _three_node_views_agree +} + +# The same shape, but the two joiners discover the cluster through a bootnode +# instead of being told about each other. +test_three_node_bootnode() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "https://gateway-1:9012" + generate_config 3 "https://gateway-1:9012" + start_node 1 || return 1 + sleep 2 + start_node 2 || return 1 + start_node 3 || return 1 + # Discovery through the bootnode is what this test is about, so wait on it + # rather than past it. Same 15s ceiling the blind settle had. + wait_for_peers 15 1 2 3 || { + log_error "the joiners did not discover the cluster through the bootnode"; return 1; } + check_debug_service 1 || { log_error "debug service not available on node 1"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key 504)" bootnode_app bootnode_inst)" \ + >/dev/null || { log_error "registration failed"; return 1; } + wait_until 20 _three_node_views_converged || true + _three_node_views_agree +} + +# The periodic persist must actually run -- and be seen to run -- and what it +# wrote must come back after a restart. +test_periodic_persistence() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local i key success=0 + for i in $(seq 1 3); do + key=$(test_public_key $((900 + i))) + verify_register_response "$(debug_register_cvm 1 "$key" "persist_app$i" "persist_inst$i")" \ + >/dev/null 2>&1 && success=$((success + 1)) + done + [ "$success" -eq 3 ] || { log_error "registered only $success/3 clients"; return 1; } + + local keys_before keys_after log wal_size + keys_before=$(get_n_keys 1) + # `persist_interval` is 5s, so the line is due in one round -- but a blind + # wait of a round and a half is both slower than it needs to be and, on a + # loaded machine, shorter than it needs to be. Poll for it. + wait_until 20 _node1_logged_a_periodic_persist || true + + log=$(dump_log 1) + grep -q "periodic persist completed" "$log" || { + log_error "periodic persist message not found in the log"; return 1; } + + wal_size=$(data_op "stat -c%s /data/node1/wavekv/node_1.wal 2>/dev/null || echo 0" | tr -d '\r') + [ "${wal_size:-0}" -gt 0 ] 2>/dev/null || { log_error "WAL file missing or empty"; return 1; } + log_info "WAL size after periodic persist: ${wal_size} bytes" + + stop_node 1 + start_node 1 || return 1 + keys_after=$(get_n_keys 1) + [ "$keys_after" -ge "$keys_before" ] || { + log_error "keys_before=$keys_before keys_after=$keys_after"; return 1; } +} + +# ------------------------------------------------------------------ admin + +test_admin_set_node_url() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local new_url="https://new-node2.example.com:8011" response stored + response=$(admin_set_node_url 1 2 "$new_url") + echo "$response" | grep -q '"error"' && { log_error "SetNodeUrl returned: $response"; return 1; } + + sleep 2 + stored=$(get_peer_url_from_sync 1 2) + [ "$stored" = "$new_url" ] || { + log_error "expected '$new_url', got '$stored'"; return 1; } +} + +test_admin_set_node_status() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + admin_set_node_url 1 2 "https://node2.example.com:8011" >/dev/null + sleep 1 + + local response + for state in down up; do + response=$(admin_set_node_status 1 2 "$state") + # Absence of "error" is not enough on its own: a request that never + # reached the node yields an empty string, which contains no "error" + # either, so a total connectivity failure would pass. + [ -n "$response" ] || { + log_error "SetNodeStatus($state) returned nothing at all"; return 1; } + echo "$response" | grep -q '"error"' && { + log_error "SetNodeStatus($state) returned: $response"; return 1; } + sleep 1 + done + + # Kept as a warning, exactly as the original had it: whether an unknown + # status is refused is not what this test is here to pin down. + response=$(admin_set_node_status 1 2 invalid) + echo "$response" | grep -q '"error"' || \ + log_warn "invalid status was not rejected (may be acceptable)" + return 0 +} + +# A node marked down must not be handed to registering CVMs, and must be handed +# out again once it is back up. +test_node_status_register_exclude() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 5 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + check_debug_service 1 || { log_error "debug service not available on node 1"; return 1; } + + admin_set_node_status 1 2 down >/dev/null + sleep 2 + local response + response=$(debug_register_cvm 1 "$(test_public_key 505)" downtest_app downtest_inst) + verify_register_response "$response" >/dev/null || { log_error "registration failed"; return 1; } + response_lists_gateway "$response" 2 && { + log_error "node 2 (down) was included in the registration response"; return 1; } + + admin_set_node_status 1 2 up >/dev/null + sleep 2 + response=$(debug_register_cvm 1 "$(test_public_key 506)" uptest_app uptest_inst2) + response_lists_gateway "$response" 2 || { + log_error "node 2 (up) was not included in the registration response"; return 1; } +} + +# A node marked down must refuse to register CVMs itself. +test_node_status_register_reject() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local response + response=$(debug_register_cvm 1 "$(test_public_key 507)" upnode_app upnode_inst) + verify_register_response "$response" >/dev/null || { + log_error "registration failed while the node was up"; return 1; } + + admin_set_node_status 1 1 down >/dev/null + sleep 2 + response=$(debug_register_cvm 1 "$(test_public_key 508)" downnode_app downnode_inst) + echo "$response" | grep -qi "error" || { + log_error "registration was not rejected while the node is down"; return 1; } + + admin_set_node_status 1 1 up >/dev/null + sleep 2 + response=$(debug_register_cvm 1 "$(test_public_key 509)" backup_app backup_inst) + verify_register_response "$response" >/dev/null || { + log_error "registration failed once the node was back up"; return 1; } +} diff --git a/dstack/gateway/test-run/e2e/configs/gateway-1.toml b/dstack/gateway/test-run/e2e/configs/gateway-1.toml index dc8e6f3b3..fa7079a2c 100644 --- a/dstack/gateway/test-run/e2e/configs/gateway-1.toml +++ b/dstack/gateway/test-run/e2e/configs/gateway-1.toml @@ -12,21 +12,35 @@ ca_certs = "/var/lib/gateway/certs/gateway-ca.cert" mandatory = false [core] -kms_url = "" rpc_domain = "gateway-1" [core.admin] enabled = true port = 9016 address = "0.0.0.0" -admin_token = "e2e-admin-token" +# TEST ONLY - do not use in production; weak/hardcoded credential +auth_token = "e2e-admin-token" [core.debug] +# TEST ONLY - do not use in production; enables debug RPC insecure_enable_debug_rpc = true -insecure_skip_attestation = false port = 9015 address = "0.0.0.0" +# Verify peer quotes against the development roots the mock collateral service +# derives from the simulator's seed. Verification itself runs the normal +# production path -- this only says the anchor may come from outside the vendor +# set, which is what makes attestation testable off TDX hardware. It does not +# switch any check off; there is no longer a switch that does. +[core.attestation] +insecure_allow_external_trust_anchors = true + +[core.attestation.urls] +pccs = "http://mock-attestation:8088" + +[core.attestation.root_ca] +tdx = "/var/lib/attestation/tdx-root-ca.pem" + [core.sync] enabled = true interval = "5s" diff --git a/dstack/gateway/test-run/e2e/configs/gateway-2.toml b/dstack/gateway/test-run/e2e/configs/gateway-2.toml index c733710b2..0beae8420 100644 --- a/dstack/gateway/test-run/e2e/configs/gateway-2.toml +++ b/dstack/gateway/test-run/e2e/configs/gateway-2.toml @@ -12,21 +12,35 @@ ca_certs = "/var/lib/gateway/certs/gateway-ca.cert" mandatory = false [core] -kms_url = "" rpc_domain = "gateway-2" [core.admin] enabled = true port = 9016 address = "0.0.0.0" -admin_token = "e2e-admin-token" +# TEST ONLY - do not use in production; weak/hardcoded credential +auth_token = "e2e-admin-token" [core.debug] +# TEST ONLY - do not use in production; enables debug RPC insecure_enable_debug_rpc = true -insecure_skip_attestation = false port = 9015 address = "0.0.0.0" +# Verify peer quotes against the development roots the mock collateral service +# derives from the simulator's seed. Verification itself runs the normal +# production path -- this only says the anchor may come from outside the vendor +# set, which is what makes attestation testable off TDX hardware. It does not +# switch any check off; there is no longer a switch that does. +[core.attestation] +insecure_allow_external_trust_anchors = true + +[core.attestation.urls] +pccs = "http://mock-attestation:8088" + +[core.attestation.root_ca] +tdx = "/var/lib/attestation/tdx-root-ca.pem" + [core.sync] enabled = true interval = "5s" diff --git a/dstack/gateway/test-run/e2e/configs/gateway-3.toml b/dstack/gateway/test-run/e2e/configs/gateway-3.toml index b02a57fc9..a590cad68 100644 --- a/dstack/gateway/test-run/e2e/configs/gateway-3.toml +++ b/dstack/gateway/test-run/e2e/configs/gateway-3.toml @@ -12,21 +12,35 @@ ca_certs = "/var/lib/gateway/certs/gateway-ca.cert" mandatory = false [core] -kms_url = "" rpc_domain = "gateway-3" [core.admin] enabled = true port = 9016 address = "0.0.0.0" -admin_token = "e2e-admin-token" +# TEST ONLY - do not use in production; weak/hardcoded credential +auth_token = "e2e-admin-token" [core.debug] +# TEST ONLY - do not use in production; enables debug RPC insecure_enable_debug_rpc = true -insecure_skip_attestation = false port = 9015 address = "0.0.0.0" +# Verify peer quotes against the development roots the mock collateral service +# derives from the simulator's seed. Verification itself runs the normal +# production path -- this only says the anchor may come from outside the vendor +# set, which is what makes attestation testable off TDX hardware. It does not +# switch any check off; there is no longer a switch that does. +[core.attestation] +insecure_allow_external_trust_anchors = true + +[core.attestation.urls] +pccs = "http://mock-attestation:8088" + +[core.attestation.root_ca] +tdx = "/var/lib/attestation/tdx-root-ca.pem" + [core.sync] enabled = true interval = "5s" diff --git a/dstack/gateway/test-run/e2e/docker-compose.yml b/dstack/gateway/test-run/e2e/docker-compose.yml index 2374b0730..ae601198a 100644 --- a/dstack/gateway/test-run/e2e/docker-compose.yml +++ b/dstack/gateway/test-run/e2e/docker-compose.yml @@ -4,9 +4,16 @@ # E2E test environment for dstack-gateway certbot functionality # Uses mock services: Pebble (ACME) + mock-cf-dns-api (Cloudflare DNS) -# Uses real TDX endpoint for attestation +# Uses a test-local dstack Guest Agent simulator for certificate and app identity flows. +# Same long-lived fixture project the cluster suite uses; run-e2e.sh brings it +# up. Referencing it rather than including it keeps one instance of the seed and +# the roots derived from it, and keeps this suite's network to itself. networks: + attestation: + external: true + name: ${FIXTURE_NS:?FIXTURE_NS must be set} + certbot-test: driver: bridge ipam: @@ -14,14 +21,24 @@ networks: - subnet: 172.30.0.0/24 volumes: - pebble-certs: + dstack-socket: + external: true + name: ${FIXTURE_NS:?FIXTURE_NS must be set}-socket + attestation-roots: + external: true + name: ${FIXTURE_NS:?FIXTURE_NS must be set}-roots services: # ==================== Mock Services ==================== - # Mock Cloudflare DNS API + # Mock Cloudflare DNS API. Built from the repo rather than pulled, and the + # same image the full-stack suite builds, so both suites answer DNS the same + # way and neither depends on a prebuilt image nobody can rebuild. + # Source: tools/mock-cf-dns. mock-cf-dns-api: - image: kvin/mock-cf-dns-api:latest + build: + context: ../../../../tools/mock-cf-dns + image: dstack-e2e-mock-cf-dns:local container_name: mock-cf-dns-api networks: certbot-test: @@ -31,15 +48,24 @@ services: environment: - PORT=8080 - DEBUG=true + # The zones certbot writes into and Pebble reads back out of. + - MOCK_CF_ZONES=test0.local,test1.local,test2.local,persist0.local,persist1.local,persist2.local,selfcheck0.local + # Advertise this container as the authoritative nameserver for those + # zones, so certbot's self-check can reach it the way it reaches a real + # one -- NS for the zone, A for the answer -- instead of failing that + # lookup and quietly falling back to its system resolver. + - MOCK_CF_NS_ADDR=172.30.0.10 healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health')"] interval: 5s timeout: 3s retries: 5 - # Pebble - Let's Encrypt test server (custom build with HTTP support) + # Pebble - Let's Encrypt test server. Upstream plus a `-http` flag, so a + # client in another container can reach it by service name without trusting + # the Pebble root: github.com/kvinwang/pebble branch http-flag. pebble: - image: kvin/pebble:latest + image: kvin/pebble:v2.10.1-http container_name: pebble command: ["-http", "-dnsserver", "172.30.0.10:53"] networks: @@ -50,7 +76,15 @@ services: - "15000:15000" # Management interface environment: - PEBBLE_VA_NOSLEEP=1 - - PEBBLE_VA_ALWAYS_VALID=1 # Skip actual DNS validation for testing + # No PEBBLE_VA_ALWAYS_VALID: the mock answers DNS over TCP, which is the + # only transport Pebble uses once it is given -dnsserver, so challenges + # are validated against the records rather than waved through. + # + # CAA is not, and cannot be here: Pebble only advertises `caaIdentities` + # in its directory metadata and never queries CAA, so no CAA record in + # this harness can permit or forbid an order. Every CAA assertion below + # is against the mock's record store -- what certbot wrote -- and none of + # them is evidence that a CA would accept or refuse it. healthcheck: test: ["CMD", "wget", "-q", "--spider", "http://localhost:14000/dir"] interval: 5s @@ -64,6 +98,7 @@ services: image: ${GATEWAY_IMAGE:-dstack-gateway:test} container_name: gateway-1 networks: + attestation: certbot-test: ipv4_address: 172.30.0.21 ports: @@ -73,11 +108,15 @@ services: - "19016:9016" # Admin volumes: - ./configs/gateway-1.toml:/etc/gateway/gateway.toml:ro + - dstack-socket:/var/run/dstack + - attestation-roots:/var/lib/attestation:ro tmpfs: - /var/lib/gateway environment: - RUST_LOG=info,dstack_gateway=debug,certbot=debug - - DSTACK_AGENT_ADDRESS=https://712eab2f507b963e11144ae67218177e93ac2a24-3000.tdxlab.dstack.org:12004/ + - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock + # The fixture is a separate project now; run-e2e.sh brings it up and waits + # for it before this one starts, so it cannot be a depends_on here. depends_on: mock-cf-dns-api: condition: service_healthy @@ -89,6 +128,12 @@ services: timeout: 3s retries: 10 start_period: 30s + # Resolve through the mock, which owns the test zones and forwards + # everything else. certbot's pre-order self-check reads DNS like any + # client; without this it queries a resolver that has never heard of the + # challenge names, so the check can only ever time out. + dns: + - 172.30.0.10 cap_add: - NET_ADMIN extra_hosts: @@ -100,6 +145,7 @@ services: image: ${GATEWAY_IMAGE:-dstack-gateway:test} container_name: gateway-2 networks: + attestation: certbot-test: ipv4_address: 172.30.0.22 ports: @@ -109,11 +155,13 @@ services: - "19026:9016" # Admin volumes: - ./configs/gateway-2.toml:/etc/gateway/gateway.toml:ro + - dstack-socket:/var/run/dstack + - attestation-roots:/var/lib/attestation:ro tmpfs: - /var/lib/gateway environment: - RUST_LOG=info,dstack_gateway=debug,certbot=debug - - DSTACK_AGENT_ADDRESS=https://712eab2f507b963e11144ae67218177e93ac2a24-3000.tdxlab.dstack.org:12004/ + - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock depends_on: gateway-1: condition: service_healthy @@ -123,6 +171,12 @@ services: timeout: 3s retries: 10 start_period: 30s + # Resolve through the mock, which owns the test zones and forwards + # everything else. certbot's pre-order self-check reads DNS like any + # client; without this it queries a resolver that has never heard of the + # challenge names, so the check can only ever time out. + dns: + - 172.30.0.10 cap_add: - NET_ADMIN @@ -131,6 +185,7 @@ services: image: ${GATEWAY_IMAGE:-dstack-gateway:test} container_name: gateway-3 networks: + attestation: certbot-test: ipv4_address: 172.30.0.23 ports: @@ -140,11 +195,13 @@ services: - "19036:9016" # Admin volumes: - ./configs/gateway-3.toml:/etc/gateway/gateway.toml:ro + - dstack-socket:/var/run/dstack + - attestation-roots:/var/lib/attestation:ro tmpfs: - /var/lib/gateway environment: - RUST_LOG=info,dstack_gateway=debug,certbot=debug - - DSTACK_AGENT_ADDRESS=https://712eab2f507b963e11144ae67218177e93ac2a24-3000.tdxlab.dstack.org:12004/ + - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock depends_on: gateway-2: condition: service_healthy @@ -154,6 +211,12 @@ services: timeout: 3s retries: 10 start_period: 30s + # Resolve through the mock, which owns the test zones and forwards + # everything else. certbot's pre-order self-check reads DNS like any + # client; without this it queries a resolver that has never heard of the + # challenge names, so the check can only ever time out. + dns: + - 172.30.0.10 cap_add: - NET_ADMIN diff --git a/dstack/gateway/test-run/e2e/run-e2e.sh b/dstack/gateway/test-run/e2e/run-e2e.sh index 835d31fb0..35fc0eb5e 100755 --- a/dstack/gateway/test-run/e2e/run-e2e.sh +++ b/dstack/gateway/test-run/e2e/run-e2e.sh @@ -10,6 +10,28 @@ set -e SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +TEST_RUN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# The attestation fixture is its own long-lived project: one simulator, one +# collateral service, one seed. Both suites reference its network and volumes +# rather than each building a copy, so the seed that signs quotes and the seed +# that derives the verifying roots cannot drift apart. +# +# Defined here, above the EXIT trap that calls it -- a cleanup handler naming a +# function defined later fails on exactly the error paths it exists for. +# The attestation fixture, namespaced to this suite. +# +# Each suite runs its own copy rather than sharing one project. The seed that +# signs quotes and the seed that derives the verifying roots come from files in +# `attestation/`, so separate instances agree by construction -- while a shared +# instance meant whichever suite finished first tore it down under the others, +# and meant the three CI workflows could not run at the same time. +FIXTURE_NS="dstack-fixture-e2e" +export FIXTURE_NS + +fixture_compose() { + docker compose -p "$FIXTURE_NS" -f "$TEST_RUN_DIR/attestation/fixture.yml" "$@" +} # Colors for output RED='\033[0;31m' @@ -20,6 +42,7 @@ NC='\033[0m' # No Color log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } log_success() { echo -e "${GREEN}[OK]${NC} $1"; } +# shellcheck disable=SC2317 # called from the EXIT trap, which shellcheck cannot see log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } log_error() { echo -e "${RED}[ERROR]${NC} $1"; } @@ -46,6 +69,10 @@ while [[ $# -gt 0 ]]; do cd "$SCRIPT_DIR" log_info "Stopping containers..." docker compose down -v --remove-orphans 2>/dev/null || true + # The fixture too: it is a project of its own, so a suite-only + # teardown leaves it and its global network and volumes behind on + # any runner that outlives the job. + fixture_compose down -v --remove-orphans 2>/dev/null || true log_success "Containers stopped" exit 0 ;; @@ -72,10 +99,39 @@ done cd "$SCRIPT_DIR" # Cleanup function +# Two runs share one compose project, one set of container names and one work +# directory, so a second run tears down what the first is using. The damage does +# not look like a collision -- it looks like flaky tests, because state vanishes +# mid-run. Refuse to start instead. +# Locked on the directory, not on a file inside it. Deleting a lock FILE does +# not release the lock, it detaches it: the holder keeps its inode while the +# next run creates a fresh one and flocks that successfully. Three runs of the +# cluster suite overlapped exactly that way -- each wiping the others' state, +# with the failures reading as flaky sync rather than as a collision -- because +# a "clean up before rerunning" step had removed the lock file by name. +exec 9<"$SCRIPT_DIR" +if ! flock -n 9; then + log_error "another run of this suite is already in progress ($SCRIPT_DIR)" + exit 1 +fi + +# Probed BEFORE the trap is installed, not next to the `up` that follows it: any +# failure in between would otherwise run cleanup with the variable unset, take +# the `:-0` default, and tear down a fixture another suite was using. +FIXTURE_WAS_UP=$(fixture_compose ps -q 2>/dev/null | wc -l) + +# shellcheck disable=SC2317 # the body runs from the EXIT trap installed below cleanup() { if ! $KEEP_RUNNING; then log_info "Stopping containers..." docker compose down -v --remove-orphans 2>/dev/null || true + # Only if this run started it. The fixture is meant to outlive a single + # suite; tearing down one this run found already up is what made the + # next run fail with "network dstack-attestation declared as external, + # but could not be found". + if [ "${FIXTURE_WAS_UP:-0}" -eq 0 ]; then + fixture_compose down -v --remove-orphans 2>/dev/null || true + fi fi } @@ -107,28 +163,16 @@ if ! $SKIP_BUILD; then fi # Step 2: Create gateway docker image (alpine for musl) +# +# Through the shared builder, not a private copy of the Dockerfile. This used to +# heredoc its own byte-identical `Dockerfile.gateway`, build from it and `rm` it +# afterwards -- which defeated the point of `build-gateway-image.sh` ("shared so +# the two suites cannot end up testing different builds"), and, because the `rm` +# was neither conditional nor trapped, left an untracked `e2e/Dockerfile.gateway` +# in the tree whenever the build failed under `set -e`. log_info "Creating gateway docker image..." cd "$SCRIPT_DIR" - -cat > Dockerfile.gateway << 'EOF' -FROM alpine:latest - -RUN apk add --no-cache \ - wireguard-tools \ - iproute2 \ - curl \ - ca-certificates - -COPY dstack-gateway /usr/local/bin/dstack-gateway - -RUN chmod +x /usr/local/bin/dstack-gateway && \ - mkdir -p /etc/gateway/certs /var/lib/gateway - -ENTRYPOINT ["/usr/local/bin/dstack-gateway", "-c", "/etc/gateway/gateway.toml"] -EOF - -docker build -t dstack-gateway:test -f Dockerfile.gateway . -rm Dockerfile.gateway +"$TEST_RUN_DIR/build-gateway-image.sh" "$SCRIPT_DIR" --skip-build log_success "Gateway image created: dstack-gateway:test" # Step 3: Run docker compose @@ -136,6 +180,24 @@ log_info "Starting e2e test environment..." export GATEWAY_IMAGE=dstack-gateway:test +# Build the mock rather than trusting whatever carries its tag. `image:` plus +# `build:` means compose reuses a local tag if one exists, and this tag is +# shared with the full-stack suite, so a stale copy from an older checkout +# would be picked up silently -- and a mock that does not answer TCP fails +# every challenge now that Pebble actually validates them. +# Build every image rather than trusting whatever carries its tag. `image:` plus +# `build:` means compose reuses a local tag if one exists, and these tags are +# shared with other suites and other checkouts, so a stale copy would be picked +# up silently -- a mock that does not answer TCP fails every challenge now that +# Pebble validates them, and a stale simulator issues certificates from an older +# cert-client, which is the code path the cluster's app_id pinning depends on. +log_info "Building the mock services and the guest agent simulator..." +docker compose build +fixture_compose build + +log_info "Starting the attestation fixture..." +fixture_compose up -d --wait + docker compose up -d mock-cf-dns-api pebble log_info "Waiting for mock services to be healthy..." sleep 5 @@ -146,8 +208,11 @@ sleep 10 # Step 4: Run tests log_info "Running tests..." -docker compose run --rm test-runner -TEST_EXIT_CODE=$? +# `set -e` is on, so a plain call would abort the script the moment the suite +# fails and the report below would be unreachable -- the failure still exits +# non-zero, but silently. Capture the status instead of inheriting it. +TEST_EXIT_CODE=0 +docker compose run --rm test-runner || TEST_EXIT_CODE=$? # Step 5: Report result (cleanup handled by trap) if [ $TEST_EXIT_CODE -eq 0 ]; then diff --git a/dstack/gateway/test-run/e2e/test.sh b/dstack/gateway/test-run/e2e/test.sh index 1ed9df3fa..66c224431 100755 --- a/dstack/gateway/test-run/e2e/test.sh +++ b/dstack/gateway/test-run/e2e/test.sh @@ -22,7 +22,7 @@ GATEWAY_PROXIES="gateway-1:9014 gateway-2:9014 gateway-3:9014" GATEWAY_DEBUG_URLS="http://gateway-1:9015 http://gateway-2:9015 http://gateway-3:9015" GATEWAY_ADMIN="http://gateway-1:9016" -# Must match `admin_token` in configs/gateway-*.toml +# Must match `auth_token` in configs/gateway-*.toml ADMIN_TOKEN="e2e-admin-token" ADMIN_AUTH_HEADER="Authorization: Bearer ${ADMIN_TOKEN}" @@ -33,6 +33,31 @@ PEBBLE_DIR="http://pebble:14000/dir" # Certificate domains to test (base domains, certs will be issued for *.domain) CERT_DOMAINS="test0.local test1.local test2.local" +# A dns-persist-01 domain. Kept out of CERT_DOMAINS because the certificate +# phases check TLS through the proxy, which needs an app behind the domain; +# here the certificate is checked through the admin API instead. +PERSIST_DOMAIN="persist0.local" + +# Mock zone ID, as server.py derives it from the zone name. +PERSIST_ZONE_ID="zone-persist0-local" + +# A second dns-persist-01 domain, used for the cases that must NOT issue. Kept +# apart from PERSIST_DOMAIN so a deliberately unanswerable order cannot be +# confused with the working one. +PERSIST_NEG_DOMAIN="persist1.local" +PERSIST_NEG_ZONE_ID="zone-persist1-local" + +# A third domain, for the second refusal. Deleting a ZT domain deliberately +# keeps its certificate ("kept for historical purposes"), so reusing a domain +# that has already issued would report the old certificate and read as a +# refusal that did not happen. +PERSIST_NEG2_DOMAIN="persist2.local" +PERSIST_NEG2_ZONE_ID="zone-persist2-local" + +# Pebble answers to this name, not to letsencrypt.org, and a validation record +# naming a CA the challenge does not list is ignored. +PERSIST_ISSUER="pebble.letsencrypt.org" + # Cloudflare mock settings CF_API_TOKEN="test-token" CF_API_URL="http://mock-cf-dns-api:8080/client/v4" @@ -44,9 +69,14 @@ TESTS_FAILED=0 # ==================== Logging ==================== -log_info() { printf "${BLUE}[INFO]${NC} %s\n" "$1"; } -log_warn() { printf "${YELLOW}[WARN]${NC} %s\n" "$1"; } -log_error() { printf "${RED}[ERROR]${NC} %s\n" "$1"; } +# Diagnostics go to stderr. `run_test` reads a test's exit code out of a command +# substitution, which captures stdout -- so a test that logs on stdout is +# reported failed no matter what it returns, and a `log_error` explaining a real +# failure is swallowed into the comparison instead of being shown. Keeping these +# off stdout fixes both, and means no test has to remember to redirect. +log_info() { printf "${BLUE}[INFO]${NC} %s\n" "$1" >&2; } +log_warn() { printf "${YELLOW}[WARN]${NC} %s\n" "$1" >&2; } +log_error() { printf "${RED}[ERROR]${NC} %s\n" "$1" >&2; } log_success() { printf "${GREEN}[PASS]${NC} %s\n" "$1"; } log_fail() { printf "${RED}[FAIL]${NC} %s\n" "$1"; } @@ -113,6 +143,390 @@ get_cert_san() { get_cert_pem "$1" "$2" | openssl x509 -noout -ext subjectAltName 2>/dev/null } +# ==================== dns-persist-01 Helpers ==================== + +admin_post() { + curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.$1" \ + -H "${ADMIN_AUTH_HEADER}" \ + -H "Content-Type: application/json" \ + -d "$2" +} + +# A dns-persist-01 domain must be accepted without any DNS credential: holding +# none is the entire point of the challenge. +test_persist_domain_needs_no_credential() { + # Idempotent: the KV store outlives a run, and an "already exists" rejection + # would mask whether the add is accepted at all. + admin_post DeleteZtDomain '{"domain": "'"${PERSIST_DOMAIN}"'"}' > /dev/null 2>&1 || true + admin_post AddZtDomain \ + '{"domain": "'"${PERSIST_DOMAIN}"'", "port": 443, "challenge": "dns-persist-01"}' \ + > /dev/null +} + +# The record is the setup under this challenge, so the API has to render it. +test_persist_domain_reports_its_record() { + local info + info=$(admin_post GetZtDomain '{"domain": "'"${PERSIST_DOMAIN}"'"}') || return 1 + # Fixed strings throughout: as a regex, "_validation-persist.*.domain" + # matches the very name it is meant to rule out. + # + # The record sits on the base name. ACME strips the wildcard before it + # creates the authorization, so a record published under the "*." form is + # never read. + if echo "$info" | grep -qF "_validation-persist.*.${PERSIST_DOMAIN}"; then + return 1 + fi + echo "$info" | grep -qF "_validation-persist.${PERSIST_DOMAIN}. IN TXT" || return 1 + echo "$info" | grep -qF "accounturi=" || return 1 + # policy=wildcard, because the gateway only ever orders *.{domain}. + echo "$info" | grep -qF "policy=wildcard" +} + +# UpdateZtDomain replaces the whole record, and `challenge` is optional so that +# a caller which does not know the field leaves it alone. A client that predates +# it -- a cached dashboard bundle, a script, an older SDK -- must not be able to +# downgrade the domain to dns-01, which its published CAA would then refuse. +test_persist_challenge_survives_an_edit() { + admin_post UpdateZtDomain \ + '{"domain": "'"${PERSIST_DOMAIN}"'", "port": 443, "priority": 5}' > /dev/null || return 1 + local info + info=$(admin_post GetZtDomain '{"domain": "'"${PERSIST_DOMAIN}"'"}') || return 1 + echo "$info" | grep -q '"priority":5' || return 1 + echo "$info" | grep -q "dns-persist-01" +} + +# Publish the validation record the way a zone owner would: once, by hand, +# before the first order. certbot never writes it -- that is the whole point -- +# so the harness stands in for the operator here. +# +# The record has to be the one the gateway reports, byte for byte, because that +# is what Pebble parses as an RFC 8659 issue-value and matches against its own +# issuer name and the requesting account. +publish_persist_record() { + local info rdata + info=$(admin_post GetZtDomain '{"domain": "'"${PERSIST_DOMAIN}"'"}') || return 1 + # "_validation-persist.<domain>. IN TXT \"<rdata>\"" -> <rdata> + rdata=$(echo "$info" \ + | tr ',' '\n' \ + | grep -F "_validation-persist.${PERSIST_DOMAIN}. IN TXT" \ + | sed -e 's/.*IN TXT \\"//' -e 's/\\".*//') + if [ -z "$rdata" ]; then + log_error "no validation record reported for ${PERSIST_DOMAIN}" >&2 + return 1 + fi + # stderr, not stdout: run_test captures the function's stdout and compares + # the whole of it against "0". + log_info "Publishing: _validation-persist.${PERSIST_DOMAIN} TXT ${rdata}" >&2 + curl -sf -X POST "${MOCK_CF_API}/client/v4/zones/${PERSIST_ZONE_ID}/dns_records" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{ + "type": "TXT", + "name": "_validation-persist.'"${PERSIST_DOMAIN}"'", + "content": "'"${rdata}"'", + "ttl": 1 + }' > /dev/null +} + +# The point of the whole exercise: a certificate is issued for a domain the +# gateway holds no credential for, and the CA validates the published record +# rather than being told to skip it -- Pebble runs without +# PEBBLE_VA_ALWAYS_VALID here. +test_persist_domain_issues_a_certificate() { + admin_post RenewZtDomainCert \ + '{"domain": "'"${PERSIST_DOMAIN}"'", "force": true}' > /dev/null 2>&1 || true + + # The renewal outlives the request that triggered it, and spends the + # advisory DNS wait -- half of renew_timeout -- before it reaches the CA. + local i=0 + while [ $i -lt 60 ]; do + if admin_post GetZtDomain '{"domain": "'"${PERSIST_DOMAIN}"'"}' \ + | grep -q '"has_cert":true'; then + return 0 + fi + sleep 2 + i=$((i + 1)) + done + return 1 +} + +# ---- Cases that must NOT issue ------------------------------------------- +# +# These are the reason the mock answers DNS at all. With PEBBLE_VA_ALWAYS_VALID +# the CA accepts any challenge, so a wrong record and a right one are +# indistinguishable and the record's grammar is pinned only by unit tests +# written against a reading of the CA. Here the CA reads it. + +# Publish one TXT record verbatim at the negative domain's validation name, +# replacing whatever is there. +publish_neg_record() { + local domain="$1" zone="$2" rdata="$3" + local records id + records=$(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null || echo '{"records":[]}') + for id in $(echo "$records" \ + | tr '{' '\n' \ + | grep -F "_validation-persist.${domain}" \ + | sed -e 's/.*"id": "//' -e 's/".*//'); do + curl -sf -X DELETE \ + "${MOCK_CF_API}/client/v4/zones/${zone}/dns_records/${id}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" > /dev/null 2>&1 || true + done + curl -sf -X POST "${MOCK_CF_API}/client/v4/zones/${zone}/dns_records" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{ + "type": "TXT", + "name": "_validation-persist.'"${domain}"'", + "content": "'"${rdata}"'", + "ttl": 1 + }' > /dev/null +} + +# The account URI the cluster actually holds, read off the domain the gateway +# already renders records for. +persist_account_uri() { + admin_post GetZtDomain '{"domain": "'"${PERSIST_DOMAIN}"'"}' \ + | tr ',' '\n' \ + | grep -F "accounturi=" \ + | head -1 \ + | sed -e 's/.*accounturi=//' -e 's/[";\\].*//' +} + +# Try to issue for the negative domain and report whether a certificate +# appeared. Deliberately short: the interesting outcome is "no certificate", +# and an order that is going to be refused says so in one pass. +neg_domain_issued() { + local domain="$1" + admin_post RenewZtDomainCert \ + '{"domain": "'"${domain}"'", "force": true}' > /dev/null 2>&1 || true + local i=0 + while [ $i -lt 45 ]; do + if admin_post GetZtDomain '{"domain": "'"${domain}"'"}' \ + | grep -q '"has_cert":true'; then + return 0 + fi + sleep 2 + i=$((i + 1)) + done + return 1 +} + +# The gateway only ever orders `*.{domain}`, so every authorization it answers +# is a wildcard one -- and the draft has a CA accept a wildcard authorization +# only from a record carrying `policy=wildcard`. A record that is correct in +# every other respect must therefore not be enough. +test_persist_record_without_wildcard_policy_is_refused() { + admin_post DeleteZtDomain '{"domain": "'"${PERSIST_NEG_DOMAIN}"'"}' > /dev/null 2>&1 || true + admin_post AddZtDomain \ + '{"domain": "'"${PERSIST_NEG_DOMAIN}"'", "port": 443, "challenge": "dns-persist-01"}' \ + > /dev/null || return 1 + local uri + uri=$(persist_account_uri) || return 1 + [ -n "$uri" ] || return 1 + publish_neg_record "${PERSIST_NEG_DOMAIN}" "${PERSIST_NEG_ZONE_ID}" \ + "${PERSIST_ISSUER}; accounturi=${uri}" || return 1 + # Inverted: issuing here would mean the policy parameter is not enforced. + ! neg_domain_issued "${PERSIST_NEG_DOMAIN}" +} + +# Adding the one parameter that was missing, and nothing else, has to flip the +# outcome. That is what makes the previous case a statement about `policy` +# rather than about some unrelated breakage. +test_persist_record_with_wildcard_policy_issues() { + local uri + uri=$(persist_account_uri) || return 1 + publish_neg_record "${PERSIST_NEG_DOMAIN}" "${PERSIST_NEG_ZONE_ID}" \ + "${PERSIST_ISSUER}; accounturi=${uri}; policy=wildcard" || return 1 + neg_domain_issued "${PERSIST_NEG_DOMAIN}" +} + +# The account URI in the record is the whole authorization. A record naming a +# different account must not let this one issue -- otherwise publishing a record +# authorizes anybody, and the challenge proves nothing. +test_persist_record_for_another_account_is_refused() { + admin_post DeleteZtDomain '{"domain": "'"${PERSIST_NEG2_DOMAIN}"'"}' > /dev/null 2>&1 || true + admin_post AddZtDomain \ + '{"domain": "'"${PERSIST_NEG2_DOMAIN}"'", "port": 443, "challenge": "dns-persist-01"}' \ + > /dev/null || return 1 + publish_neg_record "${PERSIST_NEG2_DOMAIN}" "${PERSIST_NEG2_ZONE_ID}" \ + "${PERSIST_ISSUER}; accounturi=${ACME_URL%/dir}/my-account/0; policy=wildcard" || return 1 + ! neg_domain_issued "${PERSIST_NEG2_DOMAIN}" +} + +# ---- The pre-order DNS self-check ----------------------------------------- +# +# certbot resolves the challenge name itself before telling the CA to go and +# look, so that a record that has not propagated is reported by name instead of +# as an order failure. The check is advisory: it warns and proceeds either way, +# which is exactly why nothing downstream reveals whether it worked. The mock +# logs the questions it is asked, so the check is observed directly. + +dns_queries_for() { + curl -sf "${MOCK_CF_API}/api/dns-queries" 2>/dev/null \ + | tr '{' '\n' \ + | grep -F "\"name\": \"$1\"" || true +} + +answered_queries_for() { + dns_queries_for "$1" | grep -cvF '"answers": 0' +} + +# The happy path, observed rather than triggered. By the time this runs the +# dns-persist-01 domain has issued from a record this suite published, so the +# self-check must have resolved it -- and an answered question is the only +# direct evidence, because the check warns and proceeds either way. +# +# Deliberately passive: forcing another renewal would race the periodic one, +# which picks a domain up as soon as it is added and can leave nothing for the +# forced run to do. +test_self_check_resolves_a_published_record() { + [ "$(answered_queries_for "_validation-persist.${PERSIST_DOMAIN}")" -gt 0 ] +} + +# The self-check does not ask its system resolver for the challenge record: a +# resolver that cached NXDOMAIN moments before the record was written answers +# from that cache for the whole wait, so the check reads the authoritative +# servers instead. Getting there is two questions -- NS for the zone, then A for +# the name that answers -- and if either goes unanswered certbot logs a warning +# and silently falls back to the very resolver the path exists to avoid. The +# fallback is deliberate and issuance still works, which is exactly why nothing +# else here would notice that the path never ran. +test_self_check_reads_the_authoritative_nameservers() { + # `_validation-persist.<zone>` is not a zone cut, so certbot strips the + # leading label and asks at the apex. That strip is what makes one code path + # serve both challenges, and this is where it is exercised for real. + [ "$(answered_queries_for "${PERSIST_DOMAIN}")" -gt 0 ] || return 1 + [ "$(answered_queries_for "ns.${PERSIST_DOMAIN}")" -gt 0 ] +} + +# The unhappy path, and the reason the check is advisory at all. A name with no +# record has to be polled and given up on -- the record may still be +# propagating -- rather than asked once and abandoned. +# +# Its own domain, and one nothing else queries, so this needs no clearing and +# does not care what ran before it. +test_self_check_gives_up_on_a_missing_record() { + local domain="selfcheck0.local" + local name="_validation-persist.${domain}" + admin_post DeleteZtDomain '{"domain": "'"${domain}"'"}' > /dev/null 2>&1 || true + # dns-persist-01 because nothing writes its record: the name stays empty for + # the whole wait without the test racing certbot's own cleanup. + admin_post AddZtDomain \ + '{"domain": "'"${domain}"'", "port": 443, "challenge": "dns-persist-01"}' \ + > /dev/null || return 1 + admin_post RenewZtDomainCert \ + '{"domain": "'"${domain}"'", "force": true}' > /dev/null 2>&1 || true + + local i=0 asked=0 + while [ $i -lt 45 ]; do + asked=$(dns_queries_for "$name" | wc -l) + [ "$asked" -ge 3 ] && break + sleep 2 + i=$((i + 1)) + done + # Polled, not asked once: the retry loop is what makes the wait a wait. + [ "$asked" -ge 3 ] || return 1 + # Every one a miss, or the record was not actually absent. + [ "$(answered_queries_for "$name")" -eq 0 ] +} + +# ---- Gateway operations that change shape for such a domain --------------- + +# SetCaa reconciles CAA through the DNS provider, which the gateway has no +# credential for here. It must skip the domain rather than fail the call, and +# must not write anything. +test_set_caa_skips_a_persist_domain() { + admin_post SetCaa '{}' > /dev/null 2>&1 || true + local records + records=$(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null || echo "[]") + # Split into records first and require both fields within one of them. The + # mock serializes with sorted keys, so "type" never follows "name" and a + # single fixed pattern spanning the two can only ever fail to match -- which + # in a negative assertion is a test that passes no matter what. + ! echo "$records" \ + | tr '{' '\n' \ + | grep -F "\"name\": \"${PERSIST_DOMAIN}\"" \ + | grep -qF '"type": "CAA"' +} + +# A run of SetCaa that dies between installing the `;` guards and dropping them +# leaves the guards behind, and the operator is told to rerun. The rerun has to +# work -- re-adding a byte-identical guard is what a provider that refuses +# duplicates rejects -- and it has to get there without ever lifting the +# deny-all, because a name with no issuer CAA is one any CA may issue for. +# +# The stranded state is planted directly: that is exactly what the dead run +# left, and it needs no way to kill a run mid-flight. +test_caa_rerun_recovers_without_lifting_deny_all() { + local domain="${CERT_DOMAINS%% *}" + local zone="zone-${domain//./-}" + local id tag + + # Plant the guards a dead run would have left, and remove the real records + # it had already deleted by that point. + for id in $(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null \ + | tr '{' '\n' \ + | grep -F "\"name\": \"${domain}\"" \ + | grep -F '"type": "CAA"' \ + | sed -e 's/.*"id": "//' -e 's/".*//'); do + curl -sf -X DELETE "${MOCK_CF_API}/client/v4/zones/${zone}/dns_records/${id}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" > /dev/null 2>&1 || true + done + for tag in issue issuewild; do + curl -sf -X POST "${MOCK_CF_API}/client/v4/zones/${zone}/dns_records" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"type": "CAA", "name": "'"${domain}"'", "content": "0 '"${tag}"' \";\"", "ttl": 60}' \ + > /dev/null || return 1 + done + + # From here the name is deny-all, and must stay that way. + curl -sf -X DELETE "${MOCK_CF_API}/api/caa-gaps" > /dev/null 2>&1 || true + admin_post SetCaa '{}' > /dev/null || return 1 + + # Recovered: real issuer records, and none of the guards left behind. + local caa + caa=$(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null \ + | tr '{' '\n' \ + | grep -F "\"name\": \"${domain}\"" \ + | grep -F '"type": "CAA"') + echo "$caa" | grep -qF 'accounturi=' || return 1 + # An `if`, not `&& return 1`: under `set -e` a failing left-hand side makes + # the whole list non-zero and aborts the function. + if echo "$caa" | grep -qF '0 issue \";\"'; then + return 1 + fi + + # And never fell open on the way. This is the half that separates reusing + # the stranded guard from deleting it and adding a fresh one: both end here, + # only one of them stays denied throughout. + ! curl -sf "${MOCK_CF_API}/api/caa-gaps" 2>/dev/null | grep -qF "\"${domain}\"" +} + +# Rotation registers a new account, and every published record names the old +# one. The response has to carry the replacements, because the gateway cannot +# publish them and nothing else reports them. +test_rotation_reports_the_records_to_republish() { + local out + out=$(admin_post RotateAcmeCredentials '{}') || return 1 + echo "$out" | grep -qF "_validation-persist.${PERSIST_DOMAIN}" || return 1 + # Naming the new account, not the one it replaced. + local uri + uri=$(echo "$out" | sed -e 's/.*"account_uri": *"//' -e 's/".*//') + [ -n "$uri" ] || return 1 + echo "$out" | grep -qF "accounturi=${uri}" +} + +# The claim the whole challenge exists for: certbot reads DNS and never writes +# it. Scoped to the record certbot would have written -- the zone is not empty, +# because the harness published the validation record above standing in for the +# operator, and that one is the point rather than a violation. +test_persist_domain_never_touches_the_provider() { + local records + records=$(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null || echo "[]") + ! echo "$records" | grep -qF "_acme-challenge.${PERSIST_DOMAIN}" +} + # ==================== Test Functions ==================== test_http_health() { @@ -132,7 +546,7 @@ test_certificates_match() { for proxy in $GATEWAY_PROXIES; do eval "serial${i}=\"\$(get_cert_serial \"\$proxy\" \"\$sni\")\"" - log_info "Gateway $i cert serial ($sni): $(eval echo \$serial$i)" >&2 + log_info "Gateway $i cert serial ($sni): $(eval echo \$serial$i)" i=$((i + 1)) done @@ -164,17 +578,31 @@ test_proxy_tls_health() { setup_certbot_config() { log_info "Configuring certbot via Admin API..." - # Set ACME URL - log_info "Setting ACME URL: ${ACME_URL}" + # Set ACME URL and the name this CA is known by. The issuer name is what a + # dns-persist-01 record has to carry, and it is also what CAA names for + # dns-01, so it is one setting for the whole deployment. + # renew_timeout also bounds the pre-order DNS self-check, which is capped at + # half of it. The default 300s would have a dns-persist-01 order sit for + # 150s waiting on a record this harness never publishes. + log_info "Setting ACME URL: ${ACME_URL} (issuer ${PERSIST_ISSUER})" if ! curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.SetCertbotConfig" \ -H "${ADMIN_AUTH_HEADER}" \ -H "Content-Type: application/json" \ - -d '{"acme_url": "'"${ACME_URL}"'"}' > /dev/null; then + -d '{ + "acme_url": "'"${ACME_URL}"'", + "issuer_domain_name": "'"${PERSIST_ISSUER}"'", + "renew_timeout_secs": 60 + }' > /dev/null; then log_error "Failed to set certbot config" return 1 fi - # Create DNS credential + # Create DNS credential. + # + # max_dns_wait is short but not zero: the API rejects zero outright + # ("max_dns_wait must be greater than zero"), and a rejected credential + # leaves the cluster with none -- which fails every dns-01 test downstream + # rather than this one call. log_info "Creating DNS credential..." if ! curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.CreateDnsCredential" \ -H "${ADMIN_AUTH_HEADER}" \ @@ -186,7 +614,7 @@ setup_certbot_config() { "cf_api_url": "'"${CF_API_URL}"'", "set_as_default": true, "dns_txt_ttl": 1, - "max_dns_wait": 0 + "max_dns_wait": 5 }' > /dev/null; then log_error "Failed to create DNS credential" return 1 @@ -198,14 +626,27 @@ setup_certbot_config() { curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.AddZtDomain" \ -H "${ADMIN_AUTH_HEADER}" \ -H "Content-Type: application/json" \ - -d '{"domain": "'"${domain}"'"}' > /dev/null || true - + -d '{"domain": "'"${domain}"'", "port": 443}' > /dev/null \ + || log_warn "AddZtDomain failed for $domain (may already exist)" + + # The first domain on a fresh deployment has to create the global ACME + # account, and it holds the shared ACME lock while it does. A domain + # added inside that window is refused with "retry after it finishes" + # and nothing retries it promptly, so adding three in a tight loop + # leaves two without certificates. Retry here, as the error asks. log_info "Triggering renewal for: $domain" - curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.RenewZtDomainCert" \ - -H "${ADMIN_AUTH_HEADER}" \ - -H "Content-Type: application/json" \ - -d '{"domain": "'"${domain}"'", "force": true}' > /dev/null || \ - log_warn "Renewal request failed for $domain (may retry)" + local attempt=1 + while [ $attempt -le 10 ]; do + if curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.RenewZtDomainCert" \ + -H "${ADMIN_AUTH_HEADER}" \ + -H "Content-Type: application/json" \ + -d '{"domain": "'"${domain}"'", "force": true}' > /dev/null; then + break + fi + attempt=$((attempt + 1)) + sleep 3 + done + [ $attempt -le 10 ] || log_warn "Renewal never succeeded for $domain after 10 tries" done return 0 @@ -261,32 +702,37 @@ main() { # Phase 4: Configure certbot log_phase 4 "Configure certbot" + # Fatal, not noted in passing. Every certificate test downstream needs this + # to have worked, so carrying on turns one broken precondition into twenty + # failures that all point somewhere else -- and the run still has to wait + # out the full issuance poll before reporting any of them. if ! setup_certbot_config; then log_error "Failed to setup certbot configuration" + exit 1 fi # Phase 5: Certificate issuance log_phase 5 "Certificate issuance" - local first_domain first_sni first_proxy - first_domain=$(echo "$CERT_DOMAINS" | cut -d' ' -f1) - first_sni=$(get_test_sni "$first_domain") + local first_proxy first_proxy=$(echo "$GATEWAY_PROXIES" | cut -d' ' -f1) - log_info "Waiting for certificates (up to 120s)..." - local waited=0 - while [ $waited -lt 120 ]; do - if test_certificate_issued "$first_proxy" "$first_sni"; then - log_info "Certificate detected for $first_sni" - break - fi - sleep 5 - waited=$((waited + 5)) - log_info "Waiting... (${waited}s)" - done - - local sni wildcard + # Wait for each domain, not just the first. The orders run concurrently and + # finish in any order, so the first one landing says nothing about the rest + # -- asserting on all three the moment it does made the last domain a + # coin flip. + local sni waited for domain in $CERT_DOMAINS; do sni=$(get_test_sni "$domain") + log_info "Waiting for certificate: $domain (up to 120s)" + waited=0 + while [ $waited -lt 120 ]; do + if test_certificate_issued "$first_proxy" "$sni"; then + log_info "Certificate detected for $sni" + break + fi + sleep 5 + waited=$((waited + 5)) + done run_test "Certificate issued for $domain" \ "$(test_certificate_issued "$first_proxy" "$sni"; echo $?)" done @@ -336,6 +782,47 @@ main() { log_info "No DNS TXT records (expected if certs cached)" fi + # Phase 10: dns-persist-01 plumbing, up to but not including the CA + log_phase 10 "dns-persist-01 without a DNS credential" + run_test "Domain accepted with no DNS credential" \ + "$(test_persist_domain_needs_no_credential; echo $?)" + run_test "Challenge survives an edit that omits it" \ + "$(test_persist_challenge_survives_an_edit; echo $?)" + # The record names the ACME account, so it can only be rendered -- and + # published -- once one exists. Registering is what the first order does. + run_test "Required validation record is reported" \ + "$(test_persist_domain_reports_its_record; echo $?)" + run_test "Validation record publishes" \ + "$(publish_persist_record; echo $?)" + run_test "Certificate issues without a DNS credential" \ + "$(test_persist_domain_issues_a_certificate; echo $?)" + run_test "No DNS record is ever written for it" \ + "$(test_persist_domain_never_touches_the_provider; echo $?)" + + # What the CA refuses, now that it actually looks the record up. + run_test "A record without policy=wildcard cannot answer a wildcard order" \ + "$(test_persist_record_without_wildcard_policy_is_refused; echo $?)" + run_test "Adding policy=wildcard is what makes it answerable" \ + "$(test_persist_record_with_wildcard_policy_issues; echo $?)" + run_test "A record naming another account does not authorize" \ + "$(test_persist_record_for_another_account_is_refused; echo $?)" + + # Gateway operations that change shape for a domain it cannot write. + run_test "SetCaa skips it instead of failing or writing" \ + "$(test_set_caa_skips_a_persist_domain; echo $?)" + run_test "A stranded CAA guard is recovered without falling open" \ + "$(test_caa_rerun_recovers_without_lifting_deny_all; echo $?)" + run_test "Rotation reports the records to republish" \ + "$(test_rotation_reports_the_records_to_republish; echo $?)" + + # The pre-order self-check, both ways round. + run_test "Self-check resolves a published challenge record" \ + "$(test_self_check_resolves_a_published_record; echo $?)" + run_test "Self-check polls and gives up when the record is absent" \ + "$(test_self_check_gives_up_on_a_missing_record; echo $?)" + run_test "Self-check reads the authoritative nameservers" \ + "$(test_self_check_reads_the_authoritative_nameservers; echo $?)" + # Summary log_section "Test Summary" log_info "Passed: $TESTS_PASSED" diff --git a/dstack/gateway/test-run/proxy-e2e/Dockerfile b/dstack/gateway/test-run/proxy-e2e/Dockerfile new file mode 100644 index 000000000..f6852017b --- /dev/null +++ b/dstack/gateway/test-run/proxy-e2e/Dockerfile @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +# The proxy data-path suite runs the gateway, the origin server and the probe +# client together in one container, because they must share a network +# namespace: `insecure_localhost_backend` resolves the backend to 127.0.0.1 as +# seen by the gateway itself. +FROM debian:bookworm-slim + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + --no-install-recommends \ + python3 openssl iproute2 iputils-ping curl ca-certificates procps \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /suite +COPY dstack-gateway /usr/local/bin/dstack-gateway +RUN chmod +x /usr/local/bin/dstack-gateway +COPY test_proxy.sh /suite/test_proxy.sh +COPY proxy /suite/proxy +RUN chmod +x /suite/test_proxy.sh + +ENV GATEWAY_BIN=/usr/local/bin/dstack-gateway +ENTRYPOINT ["/suite/test_proxy.sh"] diff --git a/dstack/gateway/test-run/proxy-e2e/README.md b/dstack/gateway/test-run/proxy-e2e/README.md new file mode 100644 index 000000000..f9004880b --- /dev/null +++ b/dstack/gateway/test-run/proxy-e2e/README.md @@ -0,0 +1,34 @@ +<!-- +SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> + +SPDX-License-Identifier: Apache-2.0 +--> + +# Gateway proxy data-path suite + +Runs the gateway, the origin server and the probe client in one container. They +have to share a network namespace: `insecure_localhost_backend` resolves the +backend address to `127.0.0.1` as the gateway itself sees it. + +## Why the kTLS fallback arm runs in a second container + +One arm asserts that a gateway configured for kTLS on a kernel without the TLS +ULP falls back to userspace instead of truncating a gated transfer at the gate. +The suite used to produce that condition with `sudo rmmod tls`, which cannot +work in a container and which took the module away from the whole host. + +`notls-seccomp.json` makes `setsockopt(IPPROTO_TCP, TCP_ULP)` return +`ENOPROTOOPT` for one container instead, which is exactly what `probe_ktls()` +sees on a kernel without `CONFIG_TLS`. It touches nothing outside that +container and does not care what else on the machine is using TLS. + +Note what the profile costs: `defaultAction` is `SCMP_ACT_ALLOW`, and a +`security_opt: seccomp=` *replaces* Docker's default profile rather than +extending it. So this arm runs with one syscall filtered and everything else +permitted, where the main arm runs under the default profile. That is acceptable +for a test container on a throwaway network namespace, and it is the price of +producing the condition at all -- but the two arms are not syscall-equivalent, +and a finding that reproduces only here should be checked against that. + +A seccomp profile is fixed when a container is created, so the arm needs its +own container rather than a restart of the main one. diff --git a/dstack/gateway/test-run/proxy-e2e/docker-compose.yml b/dstack/gateway/test-run/proxy-e2e/docker-compose.yml new file mode 100644 index 000000000..595b07404 --- /dev/null +++ b/dstack/gateway/test-run/proxy-e2e/docker-compose.yml @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +# Two containers from one image. The suite restarts the gateway ~25 times with a +# different config each round; that loop stays inside a container rather than +# being handed to compose, which would turn every arm into a container create. + +# The gateway asks the guest agent for its own app id at startup. That is a unix +# socket, not a network hop, so sharing it through a volume leaves this suite's +# "everything in one network namespace" requirement untouched -- these services +# stay on network_mode: bridge and never join the fixture's network. +# +# Referenced rather than included, like the other two suites: one fixture +# project, one seed, one set of roots derived from it. +volumes: + dstack-socket: + external: true + name: ${FIXTURE_NS:?FIXTURE_NS must be set}-socket + +x-suite: &suite + build: + context: . + dockerfile: Dockerfile + image: dstack-gateway-proxy-tests:local + cap_add: + - NET_ADMIN + volumes: + - dstack-socket:/var/run/dstack + # The suite's work directory, so the per-arm gateway logs outlive the + # container and CI has something to attach when an arm fails. + - ./run:/work + environment: + - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock + # A subdirectory per arm. Both arms run the whole suite, so with one shared + # WORK the second overwrote every `gw-<label>.log` the first wrote -- and the + # uploaded artifact for a main-arm failure showed the no-ULP arm's logs. + - WORK=/work/main + - KEEP_LOGS=1 + # The gateway reads /proc/net/tls_stat to report kTLS counters. In a container + # that MIB is per network namespace, so these numbers describe this run alone + # rather than whatever else on the machine happens to be using TLS. + network_mode: bridge + +services: + # No container_name: these are driven with `compose run --rm`, which ignores + # it, names the container `<project>-<service>-run-<hash>` and deletes it on + # exit. Logs are collected from the bind-mounted work directory instead. + proxy-tests: + <<: *suite + + # Same image, same suite, one arm. `setsockopt(IPPROTO_TCP, TCP_ULP)` is made + # to fail here so the gateway sees a kernel with no TLS ULP; a seccomp profile + # is fixed at creation, which is why this cannot just be another round inside + # the container above. + proxy-tests-notls: + <<: *suite + security_opt: + - seccomp=./notls-seccomp.json + # Repeats what the anchor sets: a service-level `environment` replaces the + # anchor's rather than merging with it, so omitting these here would leave + # this arm writing its logs somewhere the host never sees. + environment: + - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock + - WORK=/work/notls + - KEEP_LOGS=1 + - GWTEST_ULP_UNAVAILABLE=1 diff --git a/dstack/gateway/test-run/proxy-e2e/notls-seccomp.json b/dstack/gateway/test-run/proxy-e2e/notls-seccomp.json new file mode 100644 index 000000000..80fb123ce --- /dev/null +++ b/dstack/gateway/test-run/proxy-e2e/notls-seccomp.json @@ -0,0 +1,14 @@ +{ + "defaultAction": "SCMP_ACT_ALLOW", + "syscalls": [ + { + "names": ["setsockopt"], + "action": "SCMP_ACT_ERRNO", + "errnoRet": 92, + "args": [ + {"index": 1, "value": 6, "op": "SCMP_CMP_EQ"}, + {"index": 2, "value": 31, "op": "SCMP_CMP_EQ"} + ] + } + ] +} diff --git a/dstack/gateway/test-run/proxy-e2e/notls-seccomp.json.license b/dstack/gateway/test-run/proxy-e2e/notls-seccomp.json.license new file mode 100644 index 000000000..7ed638e2a --- /dev/null +++ b/dstack/gateway/test-run/proxy-e2e/notls-seccomp.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> + +SPDX-License-Identifier: Apache-2.0 diff --git a/dstack/gateway/test-run/proxy-e2e/run-proxy-tests.sh b/dstack/gateway/test-run/proxy-e2e/run-proxy-tests.sh new file mode 100755 index 000000000..71620424b --- /dev/null +++ b/dstack/gateway/test-run/proxy-e2e/run-proxy-tests.sh @@ -0,0 +1,126 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +# Host-side driver for the gateway proxy data-path suite. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEST_RUN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +FIXTURE_NS="dstack-fixture-proxy" +export FIXTURE_NS + +SKIP_BUILD="" +case "${1:-}" in + --skip-build) SKIP_BUILD="--skip-build" ;; + down) + docker compose -f "$SCRIPT_DIR/docker-compose.yml" down -v --remove-orphans + # The fixture too: it is a project of its own, so a suite-only teardown + # leaves it and its global network and volumes on the runner forever. + docker compose -p "$FIXTURE_NS" -f "$TEST_RUN_DIR/attestation/fixture.yml" \ + down -v --remove-orphans >/dev/null 2>&1 || true + exit 0 ;; + -h|--help) echo "Usage: $0 [--skip-build] | down"; exit 0 ;; + "") ;; + *) echo "unknown option: $1" >&2; exit 1 ;; +esac + +# The suite needs the gateway binary and its own copies of the scripts in the +# build context. +"$TEST_RUN_DIR/build-gateway-image.sh" "$SCRIPT_DIR" $SKIP_BUILD >/dev/null +cp "$TEST_RUN_DIR/test_proxy.sh" "$SCRIPT_DIR/test_proxy.sh" +rm -rf "$SCRIPT_DIR/proxy" && cp -r "$TEST_RUN_DIR/proxy" "$SCRIPT_DIR/proxy" + +# Two runs share one compose project, one set of container names and one work +# directory, so a second run tears down what the first is using. The damage does +# not look like a collision -- it looks like flaky tests, because state vanishes +# mid-run. Refuse to start instead. +# Locked on the directory, not on a file inside it. Deleting a lock FILE does +# not release the lock, it detaches it: the holder keeps its inode while the +# next run creates a fresh one and flocks that successfully. Three runs of the +# cluster suite overlapped exactly that way -- each wiping the others' state, +# with the failures reading as flaky sync rather than as a collision -- because +# a "clean up before rerunning" step had removed the lock file by name. +mkdir -p "$SCRIPT_DIR/run" +exec 9<"$SCRIPT_DIR/run" +if ! flock -n 9; then + echo "[ERROR] another run of this suite is already in progress ($SCRIPT_DIR/run)" >&2 + exit 1 +fi + +compose() { docker compose -f "$SCRIPT_DIR/docker-compose.yml" "$@"; } + +# The attestation fixture, namespaced to this suite. +# +# Each suite runs its own copy rather than sharing one project. The seed that +# signs quotes and the seed that derives the verifying roots come from files in +# `attestation/`, so separate instances agree by construction -- while a shared +# instance meant whichever suite finished first tore it down under the others, +# and meant the three CI workflows could not run at the same time. +fixture_compose() { + docker compose -p "$FIXTURE_NS" -f "$TEST_RUN_DIR/attestation/fixture.yml" "$@" +} + +# Both arms share one work directory; wipe it so a run never reads a previous +# run's logs. Two things this cannot do: +# +# - `rm -rf` it from the host. The suite container runs as root, so the certs +# and logs it leaves behind are root-owned and the host cannot remove them. +# The whole suite died here on its second local run, before a single +# assertion, with a wall of "Permission denied". CI never saw it because a +# fresh runner has no previous run to clean up. A throwaway container has +# the privileges the host lacks. +# +# - remove the directory ITSELF. `run` is what the lock above is held on, and +# replacing it with a fresh mkdir would detach that lock exactly the way a +# deleted lock file does -- the failure this suite's lock was just changed +# to prevent. Clear the contents and keep the inode. +mkdir -p "$SCRIPT_DIR/run" +docker run --rm -v "$SCRIPT_DIR/run:/r" alpine:latest \ + find /r -mindepth 1 -delete >/dev/null 2>&1 || true + +# Probed BEFORE the trap is installed. Reading it next to the `up` further down +# meant any failure in between ran cleanup with the variable unset, took the +# `:-0` default, and tore down a fixture another suite was using. +FIXTURE_WAS_UP=$(fixture_compose ps -q 2>/dev/null | wc -l) + +cleanup() { + compose down -v --remove-orphans >/dev/null 2>&1 || true + # Only if this run started it. The fixture is meant to outlive a single + # suite; tearing down one this run found already up is what made the next + # run fail with "network dstack-attestation declared as external, but could + # not be found". + if [ "${FIXTURE_WAS_UP:-0}" -eq 0 ]; then + fixture_compose down -v --remove-orphans 2>/dev/null || true + fi >/dev/null 2>&1 || true +} +trap cleanup EXIT + +compose build +fixture_compose build >/dev/null +echo "[INFO] starting the attestation fixture" >&2 +fixture_compose up -d --wait >/dev/null + +# `set -e` is on, so a bare call aborts the script the moment an arm fails and +# `$?` is never read -- both variables could only ever be 0 and the comparison +# below was a tautology. Worse, the no-TLS-ULP arm was skipped on exactly the +# runs where something was already wrong, and that arm is the only place the +# kTLS truncation regression is covered. Capture the status instead. +echo "[INFO] running the proxy suite" >&2 +MAIN_RC=0 +compose run --rm proxy-tests || MAIN_RC=$? + +echo "[INFO] running the no-TLS-ULP arm" >&2 +NOTLS_RC=0 +compose run --rm proxy-tests-notls || NOTLS_RC=$? + +# Readable by the host: the suite runs as root in the container and writes into +# the bind-mounted work directory, so CI cannot collect what it cannot open. +docker run --rm -v "$SCRIPT_DIR/run:/r" alpine:latest \ + chmod -R a+rX /r >/dev/null 2>&1 || true + +echo "[INFO] proxy-tests exit=$MAIN_RC proxy-tests-notls exit=$NOTLS_RC" >&2 +[ "$MAIN_RC" -eq 0 ] && [ "$NOTLS_RC" -eq 0 ] diff --git a/dstack/gateway/test-run/proxy/gwconfig.py b/dstack/gateway/test-run/proxy/gwconfig.py new file mode 100755 index 000000000..40b7d0f4f --- /dev/null +++ b/dstack/gateway/test-run/proxy/gwconfig.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 +"""Emit a gateway config for one arm of the proxy integration tests. + +Every knob the suite varies is written explicitly rather than left to the +shipped defaults, so a test says what it is testing and a default change shows +up as a test change rather than as a silent shift in what was covered. + +Usage: gwconfig.py DIR [key=value ...] + + splice=off|immediate|after:<bytes>[:<duration>] + ktls=off|immediate|after:<bytes>[:<duration>] + tpc=true|false thread_per_core + rebalance=true|false connection_rebalance + idle=<duration> timeouts.idle + data_timeout=true|false timeouts.data_timeout_enabled + workers=<n> +""" + +import sys + + +def gate_section(name: str, spec: str, extra: str = "") -> str: + """Render `[core.proxy.<name>]` for one of the two gated optimisations. + + Absent section = off, empty section = engage immediately, keys = gated; + the same three states the config documents. + """ + if spec == "off": + return "" + body = f"\n[core.proxy.{name}]\n" + if spec != "immediate": + _, _, rest = spec.partition(":") + parts = rest.split(":") + body += f"after_bytes = {parts[0]}\n" + if len(parts) > 1: + body += f'after_duration = "{parts[1]}"\n' + return body + extra + + +def main(): + """Write one arm's config to stdout.""" + d = sys.argv[1].rstrip("/") + o = dict(a.split("=", 1) for a in sys.argv[2:] if "=" in a) + + cert, key = f"{d}/certs/cert.pem", f"{d}/certs/key.pem" + cfg = f"""workers = 2 +address = "127.0.0.1:{o["rpc_port"]}" +[tls] +key = "{key}" +certs = "{cert}" +[tls.mutual] +ca_certs = "{cert}" +[core] +rpc_domain = "" +set_ulimit = false +[core.debug] +insecure_localhost_backend = true +insecure_enable_debug_rpc = false +[core.admin] +enabled = true +address = "127.0.0.1:{o["admin_port"]}" +auth_token = "{o["admin_token"]}" +[core.sync] +enabled = false +node_id = 1 +data_dir = "{d}/data" +[core.wg] +public_key = "" +private_key = "" +listen_port = {o["wg_port"]} +ip = "10.90.0.1/24" +reserved_net = ["10.90.0.1/32"] +client_ip_range = "10.90.0.0/25" +config_path = "{d}/wg.conf" +interface = "{o["wg_iface"]}" +endpoint = "10.90.0.1:{o["wg_port"]}" +[core.proxy] +listen_addr = "127.0.0.1" +listen_port = {o["proxy_port"]} +base_domain = "{o["base_domain"]}" +cert_chain = "{cert}" +cert_key = "{key}" +workers = {o.get("workers", "2")} +max_connections_per_app = 0 +buffer_size = 65536 +tls_versions = ["1.2"] +thread_per_core = {o.get("tpc", "true")} +connection_rebalance = {o.get("rebalance", "true")} + +[core.proxy.timeouts] +idle = "{o.get("idle", "10m")}" +data_timeout_enabled = {o.get("data_timeout", "true")} +""" + cfg += gate_section( + "tcp_splice", o.get("splice", "off"), extra="release_idle_pipes = true\n" + ) + cfg += gate_section("ktls", o.get("ktls", "off")) + print(cfg) + + +if __name__ == "__main__": + main() diff --git a/dstack/gateway/test-run/proxy/origin.py b/dstack/gateway/test-run/proxy/origin.py new file mode 100755 index 000000000..a950d2942 --- /dev/null +++ b/dstack/gateway/test-run/proxy/origin.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 +"""Origin server for the proxy integration tests. + +Serves the same content over plain HTTP (what the gateway's TLS-terminate path +talks to) and over TLS (what the passthrough path relays to), so a single test +can compare the two paths against one implementation. + +Endpoints: + /bytes/<n> `n` bytes of a deterministic pattern + /close/<n> the same, then close the connection (no keep-alive) + /halfclose reads the request, waits for the client's EOF, *then* replies -- + the shape that used to lose its response before the splice gate + /blackhole waits for the client's EOF and then never replies, holding the + connection open -- what a half-closed request looks like to the + relay's drain phase + /flood/<n> waits for the client's EOF and then sends `n` bytes as fast as + it can -- the drain phase with the *write* side blocked, if the + client has stopped reading + /trickle/<n> `n` small records spaced out in time, i.e. token streaming + /health "ok" + +Deterministic payloads mean a test can assert on a digest without a second +fetch, and the pattern is not all-zeroes so a truncated or misaligned relay +cannot accidentally look correct. +""" + +import hashlib +import os +import socket +import ssl +import sys +import threading +import time + +PATTERN = b"dstack-gateway-proxy-test-0123456789abcdef" + + +def payload(n: int) -> bytes: + """Return `n` bytes of the deterministic pattern.""" + reps = n // len(PATTERN) + 1 + return (PATTERN * reps)[:n] + + +def digest(n: int) -> str: + """Return the sha256 of what `payload(n)` returns.""" + return hashlib.sha256(payload(n)).hexdigest() + + +def _respond(conn, body: bytes, close: bool): + head = [ + b"HTTP/1.1 200 OK", + b"Content-Type: application/octet-stream", + b"Content-Length: %d" % len(body), + ] + head.append(b"Connection: close" if close else b"Connection: keep-alive") + conn.sendall(b"\r\n".join(head) + b"\r\n\r\n" + body) + + +def _read_request(conn) -> bytes | None: + buf = b"" + while b"\r\n\r\n" not in buf: + try: + chunk = conn.recv(65536) + except (OSError, ssl.SSLError): + return None + if not chunk: + return None + buf += chunk + return buf + + +def handle(conn): + """Serve one connection until it closes.""" + try: + while True: + req = _read_request(conn) + if req is None: + return + path = req.split(b" ")[1].decode() + + if path == "/health": + _respond(conn, b"ok", close=False) + elif path.startswith("/bytes/"): + _respond(conn, payload(int(path.rsplit("/", 1)[1])), close=False) + elif path.startswith("/close/"): + _respond(conn, payload(int(path.rsplit("/", 1)[1])), close=True) + return + elif path == "/halfclose": + # Wait for the client's half-close before answering. A relay that + # treats one direction's EOF as end-of-connection drops this. + conn.settimeout(20) + try: + while conn.recv(65536): + pass + except (OSError, ssl.SSLError): + pass + _respond(conn, payload(4096), close=True) + return + elif path == "/blackhole": + # Drain the client's half of the conversation, then hold the + # connection open without answering. The relay is now in its + # post-EOF drain, waiting on us; whether it ever gives up is + # what `timeouts.idle` is supposed to decide. + conn.settimeout(600) + try: + while conn.recv(65536): + pass + except (OSError, ssl.SSLError): + pass + time.sleep(600) + return + elif path.startswith("/flood/"): + # Drain the client's half, then push far more than the sockets + # in between can buffer. If the client is not reading, the + # relay's drain blocks in its *write*, which is a different + # stall from /blackhole and has to be watched too. + size = int(path.rsplit("/", 1)[1]) + conn.settimeout(600) + try: + while conn.recv(65536): + pass + except (OSError, ssl.SSLError): + pass + try: + conn.sendall( + b"HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n" % size + ) + chunk = payload(262144) + sent = 0 + while sent < size: + conn.sendall(chunk[: min(len(chunk), size - sent)]) + sent += len(chunk) + except (OSError, ssl.SSLError): + pass + return + elif path.startswith("/trickle/"): + count = int(path.rsplit("/", 1)[1]) + conn.sendall(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n") + for _ in range(count): + conn.sendall(b"40\r\n" + payload(64) + b"\r\n") + time.sleep(0.05) + conn.sendall(b"0\r\n\r\n") + return + else: + conn.sendall(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n") + return + except (OSError, ssl.SSLError): + pass + finally: + try: + conn.close() + except OSError: + pass + + +def serve(port: int, ctx: ssl.SSLContext | None): + """Accept forever on `port`, wrapping in TLS when `ctx` is given.""" + srv = socket.socket() + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind(("127.0.0.1", port)) + srv.listen(512) + while True: + raw, _ = srv.accept() + raw.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + if ctx is not None: + try: + raw = ctx.wrap_socket(raw, server_side=True) + except (OSError, ssl.SSLError): + raw.close() + continue + threading.Thread(target=handle, args=(raw,), daemon=True).start() + + +def main(): + """Start the configured listeners and idle.""" + plain = int(os.environ.get("PLAIN_PORT", "0")) + tls = int(os.environ.get("TLS_PORT", "0")) + cert, key = os.environ["CERT"], os.environ["KEY"] + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(cert, key) + + if plain: + threading.Thread(target=serve, args=(plain, None), daemon=True).start() + if tls: + threading.Thread(target=serve, args=(tls, ctx), daemon=True).start() + print("origin ready", flush=True) + try: + while True: + time.sleep(3600) + except KeyboardInterrupt: + pass + + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "digest": + print(digest(int(sys.argv[2]))) + else: + main() diff --git a/dstack/gateway/test-run/proxy/probe.py b/dstack/gateway/test-run/proxy/probe.py new file mode 100755 index 000000000..03695e615 --- /dev/null +++ b/dstack/gateway/test-run/proxy/probe.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 +"""Client-side probes for the gateway proxy integration tests. + +Each subcommand exercises one behaviour and prints a single machine-readable +verdict line, so the shell driver stays a list of expectations rather than a +pile of parsing. + +Deliberately uses raw sockets and `ssl` rather than an HTTP client: three of the +behaviours under test (half-close, TLS close_notify, idle reaping) are invisible +to a client that hides connection lifecycle from you. +""" + +import argparse +import hashlib +import socket +import ssl +import sys +import time + +import origin +from tlsclient import HalfCloseTlsClient + + +def tls_context() -> ssl.SSLContext: + """Build a client context that hides nothing about how a connection ended.""" + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + # The gateway ships TLS 1.2 by default; pin it so a version change shows up + # as a config diff rather than as a mystery here. + ctx.maximum_version = ssl.TLSVersion.TLSv1_2 + # Ragged EOF is one of the things under test, so never paper over it. + return ctx + + +def connect(args): + """Open one TLS connection to the gateway, routed by SNI.""" + raw = socket.create_connection((args.host, args.port), timeout=args.timeout) + return tls_context().wrap_socket( + raw, server_hostname=args.sni, suppress_ragged_eofs=False + ) + + +def request(sock, path: str, sni: str, close: bool = False): + """Send one HTTP request.""" + extra = "Connection: close\r\n" if close else "" + sock.sendall(f"GET {path} HTTP/1.1\r\nHost: {sni}\r\n{extra}\r\n".encode()) + + +def read_response(sock) -> tuple[bytes, str]: + """Return (body, how_it_ended).""" + buf = b"" + try: + while b"\r\n\r\n" not in buf: + chunk = sock.recv(65536) + if not chunk: + return b"", "eof_before_headers" + buf += chunk + except ssl.SSLEOFError: + return b"", "truncated_before_headers" + + head, body = buf.split(b"\r\n\r\n", 1) + length = None + for line in head.split(b"\r\n"): + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":")[1]) + if length is None: + return body, "no_content_length" + + try: + while len(body) < length: + chunk = sock.recv(65536) + if not chunk: + return body, "clean_eof" + body += chunk + except ssl.SSLEOFError: + return body, "truncated" + return body, "complete" + + +# --- probes ----------------------------------------------------------------- + + +def probe_fetch(args): + """Check the payload survives the proxy byte for byte.""" + sock = connect(args) + request(sock, f"/bytes/{args.size}", args.sni) + body, how = read_response(sock) + sock.close() + got = hashlib.sha256(body).hexdigest() + ok = got == origin.digest(args.size) and how == "complete" + print( + f"verdict={'pass' if ok else 'FAIL'} bytes={len(body)} want={args.size} end={how}" + ) + return ok + + +def probe_halfclose(args): + """Check a client that half-closes its request still gets the response. + + Regression test for the gated relays shutting down the wrong half and + returning an empty, successful response. Needs a client that can send + `close_notify` without waiting for the peer's -- see `tlsclient`. + """ + client = HalfCloseTlsClient(args.host, args.port, args.sni, args.timeout) + try: + client.send(f"GET /halfclose HTTP/1.1\r\nHost: {args.sni}\r\n\r\n".encode()) + client.close_write() + body, how = client.read_http_response() + finally: + client.close() + ok = len(body) == 4096 and how == "complete" + print(f"verdict={'pass' if ok else 'FAIL'} bytes={len(body)} want=4096 end={how}") + return ok + + +def probe_close_notify(args): + """Check the app closing first reaches the client as an orderly TLS shutdown. + + Regression test: after kTLS offload the gateway used to close with a bare + FIN, which a strict client cannot tell from a truncation attack. + """ + sock = connect(args) + request(sock, f"/close/{args.size}", args.sni, close=True) + body, how = read_response(sock) + sock.close() + ok = len(body) == args.size and how in ("complete", "clean_eof") + print(f"verdict={'pass' if ok else 'FAIL'} bytes={len(body)} end={how}") + return ok + + +def probe_idle(args): + """Check whether a connection that goes quiet is reaped, as configured. + + Regression test: configuring splice or kTLS used to bypass the watchdog + entirely, leaving `timeouts.total` (5h) as the only bound. + """ + sock = connect(args) + request(sock, f"/bytes/{args.size}", args.sni) + body, _ = read_response(sock) + if len(body) != args.size: + print(f"verdict=FAIL setup: read {len(body)} of {args.size}") + return False + + time.sleep(args.wait) + try: + sock.settimeout(10) + request(sock, "/bytes/64", args.sni) + alive = bool(sock.recv(200)) + except (OSError, ssl.SSLError): + alive = False + finally: + sock.close() + + want_alive = args.expect == "alive" + ok = alive == want_alive + print( + f"verdict={'pass' if ok else 'FAIL'} " + f"observed={'alive' if alive else 'reaped'} expected={args.expect}" + ) + return ok + + +def probe_stalled_after_halfclose(args): + """Check a half-closed request whose backend never replies is still reaped. + + Regression test: the relays dropped out of their watchdog once one + direction hit EOF and drained the other with bare reads, so a client could + hold a connection open until `timeouts.total` -- five hours by default -- + by half-closing against a backend that accepts and stays silent. + """ + client = HalfCloseTlsClient(args.host, args.port, args.sni, args.timeout) + started = time.monotonic() + try: + client.send(f"GET /blackhole HTTP/1.1\r\nHost: {args.sni}\r\n\r\n".encode()) + client.close_write() + client.sock.settimeout(args.wait) + try: + client.recv() + reaped = True + except Exception: + reaped = False + finally: + client.close() + waited = time.monotonic() - started + print( + f"verdict={'pass' if reaped else 'FAIL'} " + f"{'reaped' if reaped else 'still open'} after {waited:.1f}s" + ) + return reaped + + +def probe_stalled_write_after_halfclose(args): + """Check a half-closed request is reaped when the *client* stops reading. + + The mirror of `stalled-halfclose`: there the backend goes silent and the + relay blocks in `read`, here the backend floods and the relay blocks in + `write` because this client never drains it. Watching only the read half + leaves this one running until `timeouts.total`. + """ + client = HalfCloseTlsClient(args.host, args.port, args.sni, args.timeout) + started = time.monotonic() + try: + client.send( + f"GET /flood/{args.size} HTTP/1.1\r\nHost: {args.sni}\r\n\r\n".encode() + ) + client.close_write() + # Deliberately read nothing: let every buffer between here and the + # backend fill, so the relay is stuck in its write. + time.sleep(args.wait) + # Now drain. A reaped connection ends after whatever was buffered; a + # live one keeps feeding us the whole flood. + client.sock.settimeout(10) + drained = 0 + reaped = False + while drained < args.size: + try: + chunk = client.recv() + except Exception: + reaped = True + break + if not chunk: + reaped = True + break + drained += len(chunk) + finally: + client.close() + waited = time.monotonic() - started + print( + f"verdict={'pass' if reaped else 'FAIL'} " + f"{'reaped' if reaped else 'still open'} after {waited:.1f}s, drained {drained}B" + ) + return reaped + + +def probe_concurrent(args): + """Check many simultaneous transfers all arrive intact. + + Exercises the parts a single-connection test cannot: the pipe pool, the + per-core balancer, and whatever the gate does under real concurrency. + """ + import concurrent.futures + + def one(_): + try: + sock = connect(args) + request(sock, f"/bytes/{args.size}", args.sni) + body, how = read_response(sock) + sock.close() + return ( + hashlib.sha256(body).hexdigest() == origin.digest(args.size) + and how == "complete" + ) + except Exception: + return False + + with concurrent.futures.ThreadPoolExecutor(max_workers=args.count) as pool: + results = list(pool.map(one, range(args.count))) + ok = all(results) + print( + f"verdict={'pass' if ok else 'FAIL'} " + f"intact={sum(results)}/{args.count} size={args.size}" + ) + return ok + + +PROBES = { + "fetch": probe_fetch, + "halfclose": probe_halfclose, + "close-notify": probe_close_notify, + "idle": probe_idle, + "stalled-halfclose": probe_stalled_after_halfclose, + "stalled-write-halfclose": probe_stalled_write_after_halfclose, + "concurrent": probe_concurrent, +} + + +def main(): + """Run one probe and exit non-zero if it failed.""" + ap = argparse.ArgumentParser() + ap.add_argument("probe", choices=sorted(PROBES)) + ap.add_argument("--host", default="127.0.0.1") + ap.add_argument("--port", type=int, required=True) + ap.add_argument("--sni", required=True) + ap.add_argument("--size", type=int, default=1024) + ap.add_argument("--count", type=int, default=16) + ap.add_argument("--wait", type=float, default=12.0) + ap.add_argument("--expect", default="reaped", choices=["alive", "reaped"]) + ap.add_argument("--timeout", type=float, default=60.0) + args = ap.parse_args() + sys.exit(0 if PROBES[args.probe](args) else 1) + + +if __name__ == "__main__": + main() diff --git a/dstack/gateway/test-run/proxy/tlsclient.py b/dstack/gateway/test-run/proxy/tlsclient.py new file mode 100644 index 000000000..c4210fb68 --- /dev/null +++ b/dstack/gateway/test-run/proxy/tlsclient.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 +"""A TLS client that can half-close. + +`ssl.SSLSocket` cannot express "I have finished sending, keep sending to me". +Its only shutdown is `unwrap()`, which sends our `close_notify` and then blocks +for the peer's -- but the peer will not send its own until it has finished +replying, which is the very thing under test. Going one level lower and doing +`shutdown(SHUT_WR)` on the socket sends a bare FIN, which mid-TLS is a +truncation rather than an orderly half-close, so the peer is right to abandon +the connection. + +Driving the TLS state machine over memory BIOs solves it: `unwrap()` on an +`SSLObject` writes the `close_notify` record into the outgoing BIO before it +raises `SSLWantReadError` waiting for the reply. Flushing that BIO and then +declining to finish the handshake is exactly a half-close -- the peer sees an +orderly end of our stream and can keep writing to us. +""" + +import socket +import ssl + + +class HalfCloseTlsClient: + """A TLS connection whose write side can be closed independently.""" + + def __init__(self, host: str, port: int, sni: str, timeout: float = 30.0): + """Connect and complete the TLS handshake over memory BIOs.""" + self.sock = socket.create_connection((host, port), timeout=timeout) + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self._incoming = ssl.MemoryBIO() + self._outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + ctx.maximum_version = ssl.TLSVersion.TLSv1_2 + self._tls = ctx.wrap_bio(self._incoming, self._outgoing, server_hostname=sni) + self._eof = False + self._run(self._tls.do_handshake) + + # --- BIO plumbing ------------------------------------------------------- + + def _flush(self) -> None: + """Send whatever the TLS engine has queued.""" + data = self._outgoing.read() + if data: + self.sock.sendall(data) + + def _fill(self) -> bool: + """Feed one chunk of wire data in. False once the peer is done.""" + data = self.sock.recv(65536) + if not data: + self._incoming.write_eof() + self._eof = True + return False + self._incoming.write(data) + return True + + def _run(self, op): + """Drive one TLS operation to completion, pumping both BIOs.""" + while True: + try: + result = op() + except ssl.SSLWantReadError: + # Always flush first: the engine often needs to *send* something + # before the peer will send what it is waiting for, and skipping + # that deadlocks the handshake. + self._flush() + if not self._fill(): + raise + except ssl.SSLWantWriteError: + self._flush() + else: + self._flush() + return result + + # --- the interesting part ---------------------------------------------- + + def close_write(self) -> None: + """Send `close_notify` without waiting for the peer's. + + This is the half-close the rest of the suite could not express. The + `SSLWantReadError` is the engine asking for the peer's reply; ignoring + it is the point, since the peer owes us a response first. The TCP FIN + goes out too, so the far side of a passthrough relay sees a genuine + half-close rather than just a TLS alert. + """ + try: + self._tls.unwrap() + except (ssl.SSLWantReadError, ssl.SSLWantWriteError): + pass + self._flush() + self.sock.shutdown(socket.SHUT_WR) + + # --- ordinary I/O ------------------------------------------------------- + + def send(self, data: bytes) -> None: + """Write application data.""" + self._run(lambda: self._tls.write(data)) + + def recv(self, size: int = 65536) -> bytes: + """Read application data. `b""` means the peer closed cleanly.""" + while True: + try: + return self._tls.read(size) + except ssl.SSLWantReadError: + if self._eof: + return b"" + if not self._fill(): + return b"" + except ssl.SSLZeroReturnError: + # The peer's own close_notify: an orderly end of its stream. + return b"" + + def read_http_response(self) -> tuple[bytes, str]: + """Read one response, returning (body, how_it_ended).""" + buf = b"" + while b"\r\n\r\n" not in buf: + chunk = self.recv() + if not chunk: + return b"", "eof_before_headers" + buf += chunk + head, body = buf.split(b"\r\n\r\n", 1) + length = None + for line in head.split(b"\r\n"): + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":")[1]) + if length is None: + return body, "no_content_length" + while len(body) < length: + chunk = self.recv() + if not chunk: + return body, "truncated" + body += chunk + return body, "complete" + + def close(self) -> None: + """Drop the connection.""" + try: + self.sock.close() + except OSError: + pass diff --git a/dstack/gateway/test-run/test_certbot.sh b/dstack/gateway/test-run/test_certbot.sh deleted file mode 100755 index 26a6ba972..000000000 --- a/dstack/gateway/test-run/test_certbot.sh +++ /dev/null @@ -1,563 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> -# -# SPDX-License-Identifier: Apache-2.0 - -# Distributed Certbot E2E test script -# Tests certificate issuance and synchronization across gateway nodes - -set -m - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$SCRIPT_DIR" - -# Show help -show_help() { - echo "Usage: $0 [OPTIONS]" - echo "" - echo "Distributed Certbot E2E Test" - echo "" - echo "Options:" - echo " --fresh Clean everything and request new certificate from ACME" - echo " --sync-only Keep existing cert, only test sync between nodes" - echo " --clean Clean all test data and exit" - echo " -h, --help Show this help message" - echo "" - echo "Default (no options): Keep ACME account, request new certificate" - echo "" - echo "Examples:" - echo " $0 # Keep account, new cert" - echo " $0 --fresh # Fresh start, new account and cert" - echo " $0 --sync-only # Test sync with existing cert" - echo " $0 --clean # Clean up all test data" -} - -# Parse arguments -MODE="default" -while [[ $# -gt 0 ]]; do - case $1 in - --fresh) - MODE="fresh" - shift - ;; - --sync-only) - MODE="sync-only" - shift - ;; - --clean) - MODE="clean" - shift - ;; - -h|--help) - show_help - exit 0 - ;; - *) - echo "Unknown option: $1" - show_help - exit 1 - ;; - esac -done - -# Load environment variables from .env -if [[ -f ".env" ]]; then - source ".env" -else - echo "ERROR: .env file not found!" - echo "" - echo "Please create a .env file with the following variables:" - echo " CF_API_TOKEN=<your cloudflare api token>" - echo " CF_ZONE_ID=<your cloudflare zone id>" - echo " TEST_DOMAIN=<domain to test, e.g., *.test.example.com>" - echo "" - echo "The domain must be managed by Cloudflare and the API token must have" - echo "permissions to manage DNS records and CAA records." - exit 1 -fi - -# Validate required environment variables -if [[ -z "$CF_API_TOKEN" ]]; then - echo "ERROR: CF_API_TOKEN is not set in .env" - exit 1 -fi - -if [[ -z "$CF_ZONE_ID" ]]; then - echo "ERROR: CF_ZONE_ID is not set in .env" - exit 1 -fi - -if [[ -z "$TEST_DOMAIN" ]]; then - echo "ERROR: TEST_DOMAIN is not set in .env" - exit 1 -fi - -GATEWAY_BIN="$SCRIPT_DIR/../../target/release/dstack-gateway" -RUN_DIR="run" -CERTS_DIR="$RUN_DIR/certs" -CA_CERT="$CERTS_DIR/gateway-ca.cert" -LOG_DIR="$RUN_DIR/logs" -CURRENT_TEST="test_certbot" - -# Let's Encrypt staging URL (for testing without rate limits) -ACME_STAGING_URL="https://acme-staging-v02.api.letsencrypt.org/directory" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } -log_error() { echo -e "${RED}[ERROR]${NC} $1"; } - -cleanup() { - log_info "Cleaning up..." - sudo pkill -9 -f "dstack-gateway.*certbot_node[12].toml" >/dev/null 2>&1 || true - sudo ip link delete certbot-test1 2>/dev/null || true - sudo ip link delete certbot-test2 2>/dev/null || true - sleep 1 - stty sane 2>/dev/null || true -} - -trap cleanup EXIT - -# Generate node config with certbot enabled -generate_certbot_config() { - local node_id=$1 - local rpc_port=$((14000 + node_id * 10 + 2)) - local wg_port=$((14000 + node_id * 10 + 3)) - local proxy_port=$((14000 + node_id * 10 + 4)) - local debug_port=$((14000 + node_id * 10 + 5)) - local wg_ip="10.0.4${node_id}.1/24" - - # Build peer config - local other_node=$((3 - node_id)) # If node_id=1, other=2; if node_id=2, other=1 - local other_rpc_port=$((14000 + other_node * 10 + 2)) - - local abs_run_dir="$SCRIPT_DIR/$RUN_DIR" - local certbot_dir="$abs_run_dir/certbot_node${node_id}" - - mkdir -p "$certbot_dir" - - cat > "$RUN_DIR/certbot_node${node_id}.toml" << EOF -log_level = "info" -address = "0.0.0.0" -port = ${rpc_port} - -[tls] -key = "${abs_run_dir}/certs/gateway-rpc.key" -certs = "${abs_run_dir}/certs/gateway-rpc.cert" - -[tls.mutual] -ca_certs = "${abs_run_dir}/certs/gateway-ca.cert" -mandatory = false - -[core] -kms_url = "" -rpc_domain = "gateway.tdxlab.dstack.org" - -[core.debug] -insecure_enable_debug_rpc = true -insecure_skip_attestation = true -port = ${debug_port} -address = "127.0.0.1" - -[core.sync] -enabled = true -interval = "5s" -timeout = "10s" -my_url = "https://localhost:${rpc_port}" -bootnode = "https://localhost:${other_rpc_port}" -node_id = ${node_id} -data_dir = "${RUN_DIR}/wavekv_certbot_node${node_id}" - -[core.certbot] -enabled = true -workdir = "${certbot_dir}" -acme_url = "${ACME_STAGING_URL}" -cf_api_token = "${CF_API_TOKEN}" -cf_zone_id = "${CF_ZONE_ID}" -auto_set_caa = true -domain = "${TEST_DOMAIN}" -renew_interval = "1h" -renew_before_expiration = "720h" -renew_timeout = "5m" - -[core.wg] -private_key = "SEcoI37oGWynhukxXo5Mi8/8zZBU6abg6T1TOJRMj1Y=" -public_key = "xc+7qkdeNFfl4g4xirGGGXHMc0cABuE5IHaLeCASVWM=" -listen_port = ${wg_port} -ip = "${wg_ip}" -reserved_net = ["10.0.4${node_id}.1/31"] -client_ip_range = "10.0.4${node_id}.1/24" -config_path = "${RUN_DIR}/wg_certbot_node${node_id}.conf" -interface = "certbot-test${node_id}" -endpoint = "127.0.0.1:${wg_port}" - -[core.proxy] -cert_chain = "${certbot_dir}/live/cert.pem" -cert_key = "${certbot_dir}/live/key.pem" -base_domain = "tdxlab.dstack.org" -listen_addr = "0.0.0.0" -listen_port = ${proxy_port} -tappd_port = 8090 -external_port = ${proxy_port} -EOF - log_info "Generated certbot_node${node_id}.toml (rpc=${rpc_port}, debug=${debug_port}, proxy=${proxy_port})" -} - -start_certbot_node() { - local node_id=$1 - local config="$RUN_DIR/certbot_node${node_id}.toml" - local log_file="${LOG_DIR}/${CURRENT_TEST}_node${node_id}.log" - - log_info "Starting certbot node ${node_id}..." - mkdir -p "$RUN_DIR/wavekv_certbot_node${node_id}" - mkdir -p "$LOG_DIR" - ( sudo RUST_LOG=info "$GATEWAY_BIN" -c "$config" > "$log_file" 2>&1 & ) - - # Wait for process to either stabilize or fail - local max_wait=30 - local waited=0 - while [[ $waited -lt $max_wait ]]; do - sleep 2 - waited=$((waited + 2)) - - if ! pgrep -f "dstack-gateway.*${config}" > /dev/null; then - # Process exited, check why - log_error "Certbot node ${node_id} exited after ${waited}s" - echo "--- Log output ---" - cat "$log_file" - echo "--- End log ---" - - # Check for rate limit error - if grep -q "rateLimited" "$log_file"; then - log_error "Let's Encrypt rate limit hit. Wait a few minutes and retry." - fi - return 1 - fi - - # Check if cert files exist (indicates successful init) - local certbot_dir="$RUN_DIR/certbot_node${node_id}" - if [[ -f "$certbot_dir/live/cert.pem" ]] && [[ -f "$certbot_dir/live/key.pem" ]]; then - log_info "Certbot node ${node_id} started and certificate obtained" - return 0 - fi - - log_info "Waiting for node ${node_id} to initialize... (${waited}s)" - done - - # Process still running but no cert yet - might still be requesting - if pgrep -f "dstack-gateway.*${config}" > /dev/null; then - log_info "Certbot node ${node_id} still running, certificate request in progress" - return 0 - fi - - log_error "Certbot node ${node_id} failed to start within ${max_wait}s" - cat "$log_file" - return 1 -} - -stop_certbot_node() { - local node_id=$1 - log_info "Stopping certbot node ${node_id}..." - sudo pkill -9 -f "dstack-gateway.*certbot_node${node_id}.toml" >/dev/null 2>&1 || true - sleep 1 -} - -# Get debug sync data from a node -debug_get_sync_data() { - local debug_port=$1 - curl -s "http://localhost:${debug_port}/prpc/GetSyncData" \ - -H "Content-Type: application/json" \ - -d '{}' 2>/dev/null -} - -# Check if KvStore has cert data for the domain -check_kvstore_cert() { - local debug_port=$1 - local response=$(debug_get_sync_data "$debug_port") - - # The cert data would be in the persistent store - # For now, check if we can get any data - if [[ -z "$response" ]]; then - return 1 - fi - - # Check for cert-related keys in the response - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - # Check if there are any keys that start with 'cert/' - # This is a simplified check - print('ok') - sys.exit(0) -except Exception as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" 2>/dev/null -} - -# Check if proxy is using a valid certificate by connecting via TLS -check_proxy_cert() { - local proxy_port=$1 - - # Use gateway.{base_domain} as the SNI for health endpoint - local gateway_host="gateway.tdxlab.dstack.org" - - # Use openssl to check the certificate - local cert_info=$(echo | timeout 5 openssl s_client -connect "localhost:${proxy_port}" -servername "$gateway_host" 2>/dev/null) - - if [[ -z "$cert_info" ]]; then - log_error "Failed to connect to proxy on port ${proxy_port}" - return 1 - fi - - # Check if the certificate is valid (not self-signed test cert) - # For staging certs, the issuer should contain "Staging" or "(STAGING)" - local issuer=$(echo "$cert_info" | openssl x509 -noout -issuer 2>/dev/null) - - if echo "$issuer" | grep -qi "staging\|fake\|test"; then - log_info "Proxy on port ${proxy_port} is using Let's Encrypt staging certificate" - log_info "Issuer: $issuer" - return 0 - elif echo "$issuer" | grep -qi "let's encrypt\|letsencrypt"; then - log_info "Proxy on port ${proxy_port} is using Let's Encrypt certificate" - log_info "Issuer: $issuer" - return 0 - else - log_warn "Proxy on port ${proxy_port} certificate issuer: $issuer" - # Still return success if we got a certificate - return 0 - fi -} - -# Get certificate expiry from proxy health endpoint -get_proxy_cert_expiry() { - local proxy_port=$1 - # Use gateway.{base_domain} as the SNI for health endpoint - local gateway_host="gateway.tdxlab.dstack.org" - echo | timeout 5 openssl s_client -connect "localhost:${proxy_port}" -servername "$gateway_host" 2>/dev/null | \ - openssl x509 -noout -enddate 2>/dev/null | \ - cut -d= -f2 -} - -# Get certificate serial from proxy health endpoint -get_proxy_cert_serial() { - local proxy_port=$1 - local gateway_host="gateway.tdxlab.dstack.org" - echo | timeout 5 openssl s_client -connect "localhost:${proxy_port}" -servername "$gateway_host" 2>/dev/null | \ - openssl x509 -noout -serial 2>/dev/null | \ - cut -d= -f2 -} - -# Get certificate issuer from proxy -get_proxy_cert_issuer() { - local proxy_port=$1 - local gateway_host="gateway.tdxlab.dstack.org" - echo | timeout 5 openssl s_client -connect "localhost:${proxy_port}" -servername "$gateway_host" 2>/dev/null | \ - openssl x509 -noout -issuer 2>/dev/null -} - -# Wait for certificate to be issued (with timeout) -wait_for_cert() { - local proxy_port=$1 - local timeout_secs=${2:-300} # Default 5 minutes - local start_time=$(date +%s) - - log_info "Waiting for certificate to be issued (timeout: ${timeout_secs}s)..." - - while true; do - local current_time=$(date +%s) - local elapsed=$((current_time - start_time)) - - if [[ $elapsed -ge $timeout_secs ]]; then - log_error "Timeout waiting for certificate" - return 1 - fi - - # Try to get certificate info - local expiry=$(get_proxy_cert_expiry "$proxy_port") - if [[ -n "$expiry" ]]; then - log_info "Certificate detected! Expiry: $expiry" - return 0 - fi - - log_info "Waiting... (${elapsed}s elapsed)" - sleep 10 - done -} - -# ============================================================ -# Main Test -# ============================================================ - -do_clean() { - log_info "Cleaning all certbot test data..." - cleanup - sudo rm -rf "$RUN_DIR/certbot_node1" "$RUN_DIR/certbot_node2" - sudo rm -rf "$RUN_DIR/wavekv_certbot_node1" "$RUN_DIR/wavekv_certbot_node2" - sudo rm -f "$RUN_DIR/gateway-state-certbot-node1.json" "$RUN_DIR/gateway-state-certbot-node2.json" - log_info "Done." -} - -main() { - log_info "==========================================" - log_info "Distributed Certbot E2E Test" - log_info "==========================================" - log_info "Test domain: $TEST_DOMAIN" - log_info "ACME URL: $ACME_STAGING_URL" - log_info "Mode: $MODE" - log_info "" - - # Handle --clean mode - if [[ "$MODE" == "clean" ]]; then - do_clean - return 0 - fi - - # Handle --sync-only mode: check if cert exists - if [[ "$MODE" == "sync-only" ]]; then - if [[ ! -f "$RUN_DIR/certbot_node1/live/cert.pem" ]]; then - log_error "No existing certificate found. Run without --sync-only first." - return 1 - fi - log_info "Using existing certificate for sync test" - fi - - # Clean up processes and state - cleanup - - # Decide what to clean based on mode - case "$MODE" in - fresh) - # Clean everything including ACME account - log_info "Fresh mode: cleaning all data including ACME account" - sudo rm -rf "$RUN_DIR/certbot_node1" "$RUN_DIR/certbot_node2" - ;; - sync-only) - # Keep node1 cert, only clean node2 and wavekv - log_info "Sync-only mode: keeping node1 certificate" - sudo rm -rf "$RUN_DIR/certbot_node2" - ;; - *) - # Default: keep ACME account (credentials.json), clean certs - log_info "Default mode: keeping ACME account, requesting new certificate" - # Backup credentials if exists - if [[ -f "$RUN_DIR/certbot_node1/credentials.json" ]]; then - sudo cp "$RUN_DIR/certbot_node1/credentials.json" /tmp/certbot_credentials_backup.json - fi - sudo rm -rf "$RUN_DIR/certbot_node1" "$RUN_DIR/certbot_node2" - # Restore credentials - if [[ -f /tmp/certbot_credentials_backup.json ]]; then - mkdir -p "$RUN_DIR/certbot_node1" - sudo mv /tmp/certbot_credentials_backup.json "$RUN_DIR/certbot_node1/credentials.json" - fi - ;; - esac - - # Always clean wavekv and gateway state - sudo rm -rf "$RUN_DIR/wavekv_certbot_node1" "$RUN_DIR/wavekv_certbot_node2" - sudo rm -f "$RUN_DIR/gateway-state-certbot-node1.json" "$RUN_DIR/gateway-state-certbot-node2.json" - - # Generate configs - log_info "Generating node configurations..." - generate_certbot_config 1 - generate_certbot_config 2 - - # Start Node 1 first - it will request the certificate - log_info "" - log_info "==========================================" - log_info "Phase 1: Start Node 1 and request certificate" - log_info "==========================================" - - if ! start_certbot_node 1; then - log_error "Failed to start node 1" - return 1 - fi - - # Wait for certificate to be issued - local proxy_port_1=14014 - if ! wait_for_cert "$proxy_port_1" 300; then - log_error "Node 1 failed to obtain certificate" - cat "$LOG_DIR/${CURRENT_TEST}_node1.log" | tail -50 - return 1 - fi - - # Get Node 1's certificate info - local node1_serial=$(get_proxy_cert_serial "$proxy_port_1") - local node1_expiry=$(get_proxy_cert_expiry "$proxy_port_1") - log_info "Node 1 certificate serial: $node1_serial" - log_info "Node 1 certificate expiry: $node1_expiry" - - # Show certificate source logs for Node 1 - log_info "" - log_info "Node 1 certificate source:" - grep -E "cert\[|acme\[" "$LOG_DIR/${CURRENT_TEST}_node1.log" 2>/dev/null | sed 's/^/ /' - - # Start Node 2 - it should sync the certificate from Node 1 - log_info "" - log_info "==========================================" - log_info "Phase 2: Start Node 2 and verify sync" - log_info "==========================================" - - if ! start_certbot_node 2; then - log_error "Failed to start node 2" - return 1 - fi - - # Wait for Node 2 to sync and load the certificate - local proxy_port_2=14024 - sleep 10 # Give time for sync - - if ! wait_for_cert "$proxy_port_2" 60; then - log_error "Node 2 failed to obtain certificate via sync" - cat "$LOG_DIR/${CURRENT_TEST}_node2.log" | tail -50 - return 1 - fi - - # Get Node 2's certificate info - local node2_serial=$(get_proxy_cert_serial "$proxy_port_2") - local node2_expiry=$(get_proxy_cert_expiry "$proxy_port_2") - log_info "Node 2 certificate serial: $node2_serial" - log_info "Node 2 certificate expiry: $node2_expiry" - - # Show certificate source logs for Node 2 - log_info "" - log_info "Node 2 certificate source:" - grep -E "cert\[|acme\[" "$LOG_DIR/${CURRENT_TEST}_node2.log" 2>/dev/null | sed 's/^/ /' - - # Verify both nodes have the same certificate - log_info "" - log_info "==========================================" - log_info "Verification" - log_info "==========================================" - - if [[ "$node1_serial" == "$node2_serial" ]]; then - log_info "SUCCESS: Both nodes have the same certificate (serial: $node1_serial)" - else - log_error "FAILURE: Certificate mismatch!" - log_error " Node 1 serial: $node1_serial" - log_error " Node 2 serial: $node2_serial" - return 1 - fi - - # Check that proxy is actually using the certificate - check_proxy_cert "$proxy_port_1" - check_proxy_cert "$proxy_port_2" - - log_info "" - log_info "==========================================" - log_info "All tests passed!" - log_info "==========================================" - - return 0 -} - -# Run main -main -exit $? diff --git a/dstack/gateway/test-run/test_proxy.sh b/dstack/gateway/test-run/test_proxy.sh new file mode 100755 index 000000000..904aaab31 --- /dev/null +++ b/dstack/gateway/test-run/test_proxy.sh @@ -0,0 +1,470 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 +# +# Integration tests for the gateway's proxy data path. +# +# Covers what unit tests cannot: a real gateway process relaying real +# connections, across every combination of the two gated optimisations +# (`tcp_splice`, `ktls`) and both proxy paths (TLS terminate, TLS passthrough). +# +# Complements the `cluster/` suite, which covers the control plane, WaveKV and +# the handshake cache. This one is about bytes on the wire. +# +# Runs inside the proxy-e2e container, which supplies python3, openssl and ip, +# and carries NET_ADMIN so the WireGuard-named link the gateway expects at +# startup can be created in the container's own namespace. Nothing here needs +# sudo, and nothing it creates outlives the container. +# +# ./test_proxy.sh # build and run everything +# GATEWAY_BIN=/path/to/dstack-gateway ./test_proxy.sh +# KEEP_LOGS=1 ./test_proxy.sh # leave the work dir behind on success +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +CORE_DIR="$(cd "$HERE/../.." && pwd -P)" +PROXY_DIR="$HERE/proxy" +WORK="${WORK:-$(mktemp -d /tmp/dstack-gw-proxy-test.XXXXXX)}" + +# Ports are derived from one base so a busy CI machine can shift the whole set. +BASE_PORT="${BASE_PORT:-38400}" +PROXY_PORT=$((BASE_PORT + 0)) +RPC_PORT=$((BASE_PORT + 1)) +ADMIN_PORT=$((BASE_PORT + 2)) +WG_PORT=$((BASE_PORT + 3)) +ORIGIN_PLAIN=$((BASE_PORT + 4)) +ORIGIN_TLS=$((BASE_PORT + 5)) + +BASE_DOMAIN="gwtest.local" +# `insecure_localhost_backend` routes `localhost-<port>[s]` to 127.0.0.1:<port>, +# which is what lets these tests run without registering a CVM. +SNI_TERMINATE="localhost-$ORIGIN_PLAIN.$BASE_DOMAIN" +SNI_PASSTHROUGH="localhost-${ORIGIN_TLS}s.$BASE_DOMAIN" +ADMIN_TOKEN="proxy-integration-test" +WG_IFACE="${WG_IFACE:-gwtest0}" + +GATEWAY_BIN="${GATEWAY_BIN:-$CORE_DIR/target/release/dstack-gateway}" + +PASS=0 +FAIL=0 +SKIP=0 +FAILED_NAMES=() + +say() { printf '%s\n' "$*"; } +group() { printf '\n\033[1m== %s ==\033[0m\n' "$*"; } + +# $1 = test name, remaining args = command. The command must exit 0 to pass. +check() { + local name="$1"; shift + local out rc + out=$("$@" 2>&1); rc=$? + if [ $rc -eq 0 ]; then + PASS=$((PASS + 1)) + printf ' \033[32mPASS\033[0m %-52s %s\n' "$name" "$(tail -1 <<<"$out")" + else + FAIL=$((FAIL + 1)) + FAILED_NAMES+=("$name") + printf ' \033[31mFAIL\033[0m %-52s %s\n' "$name" "$(tail -1 <<<"$out")" + printf ' %s\n' "${out//$'\n'/$'\n' }" | tail -12 + fi +} + +skip() { + SKIP=$((SKIP + 1)) + printf ' \033[33mSKIP\033[0m %-52s %s\n' "$1" "$2" +} + +probe() { python3 "$PROXY_DIR/probe.py" "$@"; } + +# --- setup ------------------------------------------------------------------ + +cleanup() { + stop_gateway + [ -n "${ORIGIN_PID:-}" ] && kill "$ORIGIN_PID" 2>/dev/null + if [ -n "${WG_CREATED:-}" ]; then + ip link del "$WG_IFACE" 2>/dev/null + fi + if [ $FAIL -eq 0 ] && [ -z "${KEEP_LOGS:-}" ]; then + rm -rf "$WORK" + else + say "work dir kept at $WORK" + fi +} +trap cleanup EXIT + +require() { + command -v "$1" >/dev/null || { say "missing required tool: $1"; exit 1; } +} + +setup() { + require python3; require openssl; require ip; require ss + mkdir -p "$WORK/certs" "$WORK/logs" + + # Fail fast on a port that is already taken. A stale listener does not just + # break startup: the probes reach *it* instead, and its config is not the arm + # under test, so the run reports a scatter of unrelated assertion failures. + # That cost a long time to diagnose once. + local port + local busy="" + for port in "$PROXY_PORT" "$RPC_PORT" "$ADMIN_PORT" "$ORIGIN_PLAIN" "$ORIGIN_TLS"; do + if ss -ltn "sport = :$port" 2>/dev/null | grep -q LISTEN; then + busy="$busy $port" + fi + done + if [ -n "$busy" ]; then + say "ports already in use:$busy" + say "stop whatever holds them, or re-run with BASE_PORT set to a free range" + exit 1 + fi + + if [ ! -x "$GATEWAY_BIN" ]; then + say "building the gateway (set GATEWAY_BIN to skip)" + (cd "$CORE_DIR" && cargo build --release -p dstack-gateway) || exit 1 + fi + + openssl req -x509 -newkey rsa:2048 -nodes -days 2 \ + -keyout "$WORK/certs/key.pem" -out "$WORK/certs/cert.pem" \ + -subj "/CN=$BASE_DOMAIN" \ + -addext "subjectAltName=DNS:$BASE_DOMAIN,DNS:*.$BASE_DOMAIN,IP:127.0.0.1" \ + 2>/dev/null || { say "failed to generate a test certificate"; exit 1; } + + # The gateway insists on a link with the configured name at startup. A real + # WireGuard device also makes `wg show` work, which the Status RPC needs; a + # dummy link is enough for the data path, so fall back to one rather than + # skipping every test on a host without the wireguard module. + # NET_ADMIN in the container's own namespace, so no sudo and nothing that + # outlives the run. + if ip link show "$WG_IFACE" >/dev/null 2>&1; then + WG_KIND=preexisting + elif ip link add "$WG_IFACE" type wireguard 2>/dev/null; then + WG_KIND=wireguard; WG_CREATED=1 + elif ip link add "$WG_IFACE" type dummy 2>/dev/null; then + WG_KIND=dummy; WG_CREATED=1 + else + say "cannot create the link '$WG_IFACE' the gateway needs at startup"; exit 1 + fi + [ -n "${WG_CREATED:-}" ] && ip link set "$WG_IFACE" up + say "link $WG_IFACE: $WG_KIND" + + CERT="$WORK/certs/cert.pem" KEY="$WORK/certs/key.pem" \ + PLAIN_PORT="$ORIGIN_PLAIN" TLS_PORT="$ORIGIN_TLS" \ + python3 "$PROXY_DIR/origin.py" >"$WORK/logs/origin.log" 2>&1 & + ORIGIN_PID=$! + for _ in $(seq 50); do + grep -q "origin ready" "$WORK/logs/origin.log" 2>/dev/null && break + sleep 0.2 + done + grep -q "origin ready" "$WORK/logs/origin.log" || { say "origin failed to start"; exit 1; } +} + +# --- gateway lifecycle ------------------------------------------------------ + +stop_gateway() { + [ -n "${GW_PID:-}" ] || return 0 + kill "$GW_PID" 2>/dev/null + for _ in $(seq 50); do + kill -0 "$GW_PID" 2>/dev/null || break + sleep 0.1 + done + kill -9 "$GW_PID" 2>/dev/null + GW_PID="" + # The suite restarts the gateway ~25 times. Killing the process is not enough: + # its listeners linger briefly, and the next arm then dies with EADDRINUSE on + # the RPC port -- which showed up as unrelated assertions failing. Wait for the + # ports to actually come back before handing them to the next arm. + local port + for port in "$PROXY_PORT" "$RPC_PORT" "$ADMIN_PORT"; do + for _ in $(seq 100); do + ss -ltn "sport = :$port" 2>/dev/null | grep -q LISTEN || break + sleep 0.1 + done + done +} + +# start_gateway <label> [gwconfig key=value ...] +start_gateway() { + local label="$1"; shift + stop_gateway + rm -rf "$WORK/data"; mkdir -p "$WORK/data" + GW_LOG="$WORK/logs/gw-$label.log" + python3 "$PROXY_DIR/gwconfig.py" "$WORK" \ + proxy_port="$PROXY_PORT" rpc_port="$RPC_PORT" admin_port="$ADMIN_PORT" \ + wg_port="$WG_PORT" wg_iface="$WG_IFACE" admin_token="$ADMIN_TOKEN" \ + base_domain="$BASE_DOMAIN" "$@" >"$WORK/gw.toml" || return 1 + RUST_LOG="${RUST_LOG:-warn}" "$GATEWAY_BIN" -c "$WORK/gw.toml" \ + >"$GW_LOG" 2>&1 & + GW_PID=$! + for _ in $(seq 100); do + if probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 64 \ + >/dev/null 2>&1; then + return 0 + fi + kill -0 "$GW_PID" 2>/dev/null || { say "gateway exited; see $GW_LOG"; return 1; } + sleep 0.2 + done + say "gateway did not become ready; see $GW_LOG" + return 1 +} + +# Assert on the gateway's own log, which is where the fallbacks announce +# themselves -- a fallback that happens silently is the bug, not the fallback. +log_contains() { grep -q "$1" "$GW_LOG"; } +log_lacks() { ! grep -q "$1" "$GW_LOG"; } + +tls_stat() { awk -v k="$1" '$1==k{print $2}' /proc/net/tls_stat 2>/dev/null || echo 0; } + +# --- the arms --------------------------------------------------------------- +# +# Both gated optimisations have three states, and the point of the gates is that +# the state can change mid-connection, so each arm is run against both proxy +# paths and both sides of its gate. + +ARMS=( + "baseline|splice=off ktls=off" + "splice-immediate|splice=immediate ktls=off" + "splice-gated|splice=after:65536:5s ktls=off" + "ktls-immediate|splice=immediate ktls=immediate" + "ktls-gated|splice=after:65536:5s ktls=after:65536:5s" +) + +test_data_path() { + group "data path: payload integrity across every arm and both paths" + local arm name cfg + for arm in "${ARMS[@]}"; do + name="${arm%%|*}"; cfg="${arm#*|}" + # shellcheck disable=SC2086 # $cfg is a list of gwconfig arguments + start_gateway "$name" $cfg || { FAIL=$((FAIL+1)); FAILED_NAMES+=("$name: startup"); continue; } + # Under the gate and over it, so both halves of each arm are exercised. + check "$name / terminate 1 KiB" \ + probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 1024 + check "$name / terminate 1 MiB" \ + probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 1048576 + check "$name / passthrough 1 KiB" \ + probe fetch --port "$PROXY_PORT" --sni "$SNI_PASSTHROUGH" --size 1024 + check "$name / passthrough 1 MiB" \ + probe fetch --port "$PROXY_PORT" --sni "$SNI_PASSTHROUGH" --size 1048576 + check "$name / 16 concurrent transfers" \ + probe concurrent --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 262144 --count 16 + done +} + +test_half_close() { + group "half-close: a client that finishes its request still gets the reply" + # Needs a TLS client that can send close_notify without waiting for the + # peer's, which `ssl.SSLSocket` cannot express -- see `proxy/tlsclient.py`. + # This found a bug `shutdown(SHUT_WR)` could not: after kTLS offload, splice + # reports the client's close_notify as EINVAL, which used to be fatal and + # took the app's in-flight response down with it. + local arm name cfg + for arm in "${ARMS[@]}"; do + name="${arm%%|*}"; cfg="${arm#*|}" + # shellcheck disable=SC2086 # $cfg is a list of gwconfig arguments + start_gateway "halfclose-$name" $cfg || { FAIL=$((FAIL+1)); continue; } + check "$name / terminate" \ + probe halfclose --port "$PROXY_PORT" --sni "$SNI_TERMINATE" + check "$name / passthrough" \ + probe halfclose --port "$PROXY_PORT" --sni "$SNI_PASSTHROUGH" + done +} + +test_close_notify() { + group "close: the app closing reaches the client as an orderly TLS shutdown" + local arm name cfg + for arm in "${ARMS[@]}"; do + name="${arm%%|*}"; cfg="${arm#*|}" + # shellcheck disable=SC2086 # $cfg is a list of gwconfig arguments + start_gateway "close-$name" $cfg || { FAIL=$((FAIL+1)); continue; } + check "$name / terminate 1 MiB" \ + probe close-notify --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 1048576 + done +} + +test_idle_timeout() { + group "timeouts.idle applies to every relay path, gated or not" + # 3s window, 12s of silence: comfortably past it without making CI slow. + local common="idle=3s data_timeout=true" + local arm + for arm in \ + "buffered|splice=off ktls=off|$SNI_TERMINATE|1048576" \ + "splice engaged|splice=after:1024 ktls=off|$SNI_PASSTHROUGH|1048576" \ + "ktls engaged|splice=immediate ktls=immediate|$SNI_TERMINATE|1048576" \ + "splice configured, gate not reached|splice=after:10485760 ktls=off|$SNI_PASSTHROUGH|1024" \ + "ktls configured, gate not reached|ktls=after:10485760 splice=after:10485760|$SNI_TERMINATE|1024" + do + local name="${arm%%|*}" rest="${arm#*|}" + local cfg="${rest%%|*}" rest2="${rest#*|}" + local sni="${rest2%%|*}" size="${rest2##*|}" + # shellcheck disable=SC2086 # both are lists of gwconfig arguments + start_gateway "idle-${name// /-}" $cfg $common || { FAIL=$((FAIL+1)); continue; } + check "reaped: $name" \ + probe idle --port "$PROXY_PORT" --sni "$sni" --size "$size" --wait 12 --expect reaped + done + + # A half-closed request whose backend never answers must still be reaped: + # the drain phase runs outside the main relay loop, so it needs the watchdog + # of its own that it did not originally have. + local arm2 + for arm2 in \ + "buffered|splice=off ktls=off|$SNI_TERMINATE" \ + "splice engaged|splice=immediate ktls=off|$SNI_PASSTHROUGH" \ + "splice gated|splice=after:1048576 ktls=off|$SNI_PASSTHROUGH" \ + "ktls gated|splice=after:1048576 ktls=after:1048576|$SNI_TERMINATE" + do + local n2="${arm2%%|*}" r2="${arm2#*|}" + local c2="${r2%%|*}" s2="${r2##*|}" + # shellcheck disable=SC2086 # $c2 is a list of gwconfig arguments + start_gateway "stalled-${n2// /-}" $c2 $common || { FAIL=$((FAIL+1)); continue; } + check "reaped after half-close, backend silent: $n2" \ + probe stalled-halfclose --port "$PROXY_PORT" --sni "$s2" --wait 20 + # The mirror: the backend floods and *this* side stops reading, so the + # relay stalls in its write instead of its read. Watching only the read + # half left this one running to `timeouts.total`. + check "reaped after half-close, client not reading: $n2" \ + probe stalled-write-halfclose --port "$PROXY_PORT" --sni "$s2" \ + --size 67108864 --wait 15 --timeout 90 + done + + start_gateway "idle-disabled" splice=immediate ktls=immediate idle=3s data_timeout=false \ + || { FAIL=$((FAIL+1)); return; } + check "kept: data_timeout_enabled = false" \ + probe idle --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 1048576 --wait 12 --expect alive +} + +test_ktls_engages() { + group "kTLS actually offloads, and only when the gate says so" + if [ ! -r /proc/net/tls_stat ]; then + skip "kTLS offload counters" "/proc/net/tls_stat unavailable" + return + fi + + start_gateway "ktls-engage" splice=immediate ktls=immediate || { FAIL=$((FAIL+1)); return; } + if log_contains "kTLS is configured but unavailable"; then + skip "kTLS offload counters" "kernel has no TLS ULP; fallback path covered below" + return + fi + local before after err_before err_after + before=$(tls_stat TlsTxSw); err_before=$(tls_stat TlsDecryptError) + check "immediate / 1 MiB transfer intact" \ + probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 1048576 + after=$(tls_stat TlsTxSw); err_after=$(tls_stat TlsDecryptError) + check "immediate / offloaded to the kernel" \ + test "$after" -gt "$before" + check "immediate / no TLS decrypt errors" \ + test "$err_after" -eq "$err_before" + + start_gateway "ktls-gate" splice=after:65536 ktls=after:65536 || { FAIL=$((FAIL+1)); return; } + before=$(tls_stat TlsTxSw) + probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 1024 >/dev/null + after=$(tls_stat TlsTxSw) + check "gated / a small request does not offload" test "$after" -eq "$before" + probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 1048576 >/dev/null + after=$(tls_stat TlsTxSw) + check "gated / a large transfer does offload" test "$after" -gt "$before" +} + +test_capability_fallbacks() { + group "capability fallbacks announce themselves and keep serving" + + # kTLS on a kernel without the TLS ULP must fall back, not truncate. + # + # The condition used to be produced with `sudo rmmod tls`, which takes the + # module away from the entire host, needs passwordless sudo, and silently + # skips whenever anything else on the machine is using TLS. The suite runs in + # a container now, where a seccomp profile makes + # `setsockopt(IPPROTO_TCP, TCP_ULP)` return ENOPROTOOPT -- which is precisely + # what probe_ktls() sees on a kernel built without CONFIG_TLS, and affects + # nothing outside this container. + # + # A seccomp profile is fixed at container creation, so this arm gets its own + # container and the main run skips it. + if [ "${GWTEST_ULP_UNAVAILABLE:-0}" = 1 ]; then + start_gateway "ktls-no-ulp" splice=immediate ktls=after:65536 + check "no TLS ULP / warns at startup" log_contains "kTLS is configured but unavailable" + check "no TLS ULP / small request intact" \ + probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 1024 + # The regression this guards: a gated offload used to return HTTP 200 with + # the body cut off at exactly the gate. + check "no TLS ULP / large transfer not truncated" \ + probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 1048576 + stop_gateway + else + skip "kTLS fallback without the TLS ULP" "runs in the notls container" + fi + + start_gateway "rebalance-inert" splice=off ktls=off tpc=false rebalance=true \ + || { FAIL=$((FAIL+1)); return; } + check "connection_rebalance without thread_per_core / warns" \ + log_contains "connection_rebalance is set but has no effect" + check "connection_rebalance without thread_per_core / still serves" \ + probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 65536 + + start_gateway "rebalance-on" splice=off ktls=off tpc=true rebalance=true \ + || { FAIL=$((FAIL+1)); return; } + check "thread_per_core + rebalance / no inert warning" \ + log_lacks "connection_rebalance is set but has no effect" + check "thread_per_core + rebalance / serves concurrently" \ + probe concurrent --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 131072 --count 24 +} + +test_runtime_modes() { + group "runtime modes all serve traffic" + local mode + for mode in "tpc=true rebalance=true" "tpc=true rebalance=false" \ + "tpc=false rebalance=false"; do + # shellcheck disable=SC2086 # $mode is a pair of gwconfig arguments + start_gateway "mode-${mode// /-}" splice=off ktls=off $mode \ + || { FAIL=$((FAIL+1)); FAILED_NAMES+=("$mode: startup"); continue; } + check "$mode" \ + probe concurrent --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 65536 --count 16 + done +} + +test_accel_status() { + group "the Status RPC reports what the data path is doing" + start_gateway "accel" splice=after:65536:5s ktls=after:65536:5s || { FAIL=$((FAIL+1)); return; } + # `Status` calls `refresh_state`, which shells out to `wg show` -- that needs + # both a real WireGuard device and the privileges to query it. Probe it rather + # than guessing from the link kind, so this skips for the right reason. + if ! curl -sf -H "Authorization: Bearer $ADMIN_TOKEN" \ + "http://127.0.0.1:$ADMIN_PORT/prpc/Admin.Status?json" \ + | grep -q '"accel"'; then + skip "accel status counters" "Status RPC unavailable (wg show needs a real device + privileges)" + return + fi + probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 1048576 >/dev/null + probe fetch --port "$PROXY_PORT" --sni "$SNI_PASSTHROUGH" --size 1048576 >/dev/null + local accel + accel=$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" \ + "http://127.0.0.1:$ADMIN_PORT/prpc/Admin.Status?json" \ + | python3 -c 'import sys,json; print(json.dumps(json.load(sys.stdin).get("accel",{})))') + say " $accel" + check "reports the configured splice mode" \ + python3 -c "import sys,json; a=json.loads(sys.argv[1]); sys.exit(0 if 'after' in a.get('splice_mode','') else 1)" "$accel" + check "counts spliced connections" \ + python3 -c "import sys,json; a=json.loads(sys.argv[1]); sys.exit(0 if a.get('splice_engaged',0) > 0 else 1)" "$accel" + check "reports no failed offloads" \ + python3 -c "import sys,json; a=json.loads(sys.argv[1]); sys.exit(0 if a.get('ktls_offload_failed',0) == 0 else 1)" "$accel" +} + +# --- run -------------------------------------------------------------------- + +setup +say "gateway: $GATEWAY_BIN" +say "work dir: $WORK" + +test_data_path +test_half_close +test_close_notify +test_idle_timeout +test_ktls_engages +test_capability_fallbacks +test_runtime_modes +test_accel_status + +printf '\n\033[1m%d passed, %d failed, %d skipped\033[0m\n' "$PASS" "$FAIL" "$SKIP" +if [ $FAIL -gt 0 ]; then + printf 'failed:\n' + printf ' - %s\n' "${FAILED_NAMES[@]}" + exit 1 +fi diff --git a/dstack/gateway/test-run/test_suite.sh b/dstack/gateway/test-run/test_suite.sh deleted file mode 100755 index 36eb44ef8..000000000 --- a/dstack/gateway/test-run/test_suite.sh +++ /dev/null @@ -1,2136 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> -# -# SPDX-License-Identifier: Apache-2.0 - -# WaveKV integration test script -# -# This legacy integration script predates the repository-wide shellcheck hook. -# Keep the existing style warnings suppressed so small harness fixes do not -# require a full script rewrite. -# shellcheck disable=SC2015,SC2034,SC2086,SC2155,SC2164 - -# Don't use set -e as it causes issues with cleanup and test flow -# set -e - -# Disable job control messages (prevents "Killed" messages from messing up output) -set +m - -# Fix terminal output - ensure proper line endings -stty -echoctl 2>/dev/null || true - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$SCRIPT_DIR" - -GATEWAY_BIN="${GATEWAY_BIN:-${SCRIPT_DIR}/../../target/release/dstack-gateway}" -RUN_DIR="run" -CERTS_DIR="$RUN_DIR/certs" -CA_CERT="$CERTS_DIR/gateway-ca.cert" -LOG_DIR="$RUN_DIR/logs" -CURRENT_TEST="" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } -log_error() { echo -e "${RED}[ERROR]${NC} $1"; } - -cleanup() { - log_info "Cleaning up..." - # Kill only dstack-gateway processes started by this test (matching our specific config path) - # Use absolute path to avoid killing system dstack-gateway processes - pkill -9 -f "dstack-gateway -c ${SCRIPT_DIR}/${RUN_DIR}/node" >/dev/null 2>&1 || true - pkill -9 -f "dstack-gateway.*${SCRIPT_DIR}/${RUN_DIR}/node" >/dev/null 2>&1 || true - sleep 1 - # Only delete WireGuard interfaces with sudo (these are our test interfaces) - sudo ip link delete wavekv-test1 2>/dev/null || true - sudo ip link delete wavekv-test2 2>/dev/null || true - sudo ip link delete wavekv-test3 2>/dev/null || true - # Clean up all wavekv data directories to prevent peer list contamination - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" "$RUN_DIR/wavekv_node3" 2>/dev/null || true - rm -f "$RUN_DIR/gateway-state-node"*.json 2>/dev/null || true - sleep 1 - stty sane 2>/dev/null || true -} - -trap cleanup EXIT - -# Generate node configs -# Usage: generate_config <node_id> [bootnode_url] -generate_config() { - local node_id=$1 - local bootnode_url=${2:-""} - local rpc_port=$((13000 + node_id * 10 + 2)) - local wg_port=$((13000 + node_id * 10 + 3)) - local proxy_port=$((13000 + node_id * 10 + 4)) - local debug_port=$((13000 + node_id * 10 + 5)) - local admin_port=$((13000 + node_id * 10 + 6)) - local wg_ip="10.0.3${node_id}.1/24" - - # Use absolute paths to avoid Rocket's relative path resolution issues - local abs_run_dir="$SCRIPT_DIR/$RUN_DIR" - cat >"$RUN_DIR/node${node_id}.toml" <<EOF -log_level = "info" -address = "0.0.0.0:${rpc_port}" - -[tls] -# Use absolute paths since Rocket resolves relative paths from config file directory -key = "${abs_run_dir}/certs/gateway-rpc.key" -certs = "${abs_run_dir}/certs/gateway-rpc.cert" - -[tls.mutual] -ca_certs = "${abs_run_dir}/certs/gateway-ca.cert" -mandatory = false - -[core] -# Empty kms_url to skip auto-cert generation (we use pre-generated certs) -kms_url = "" -rpc_domain = "gateway.tdxlab.dstack.org" - -[core.debug] -insecure_enable_debug_rpc = true -insecure_skip_attestation = true -address = "127.0.0.1:${debug_port}" - -[core.admin] -enabled = true -address = "127.0.0.1:${admin_port}" -insecure_no_auth = true - -[core.sync] -enabled = true -interval = "5s" -timeout = "10s" -my_url = "https://localhost:${rpc_port}" -bootnode = "${bootnode_url}" -node_id = ${node_id} -data_dir = "${RUN_DIR}/wavekv_node${node_id}" -persist_interval = "5s" - -[core.certbot] -enabled = false - -[core.wg] -private_key = "SEcoI37oGWynhukxXo5Mi8/8zZBU6abg6T1TOJRMj1Y=" -public_key = "xc+7qkdeNFfl4g4xirGGGXHMc0cABuE5IHaLeCASVWM=" -listen_port = ${wg_port} -ip = "${wg_ip}" -reserved_net = ["10.0.3${node_id}.1/31"] -client_ip_range = "10.0.3${node_id}.1/24" -config_path = "${RUN_DIR}/wg_node${node_id}.conf" -interface = "wavekv-test${node_id}" -endpoint = "127.0.0.1:${wg_port}" - -[core.proxy] -cert_chain = "${RUN_DIR}/certbot/live/cert.pem" -cert_key = "${RUN_DIR}/certbot/live/key.pem" -base_domain = "tdxlab.dstack.org" -listen_addr = "0.0.0.0" -listen_port = ${proxy_port} -tappd_port = 8090 -external_port = ${proxy_port} -EOF - log_info "Generated node${node_id}.toml (rpc=${rpc_port}, debug=${debug_port}, admin=${admin_port}, bootnode=${bootnode_url:-none})" -} - -wait_for_port_free() { - local port=$1 - local max_wait=10 - local waited=0 - while [[ $waited -lt $max_wait ]]; do - if ! netstat -tlnp 2>/dev/null | grep -q ":${port} "; then - return 0 - fi - sleep 1 - ((waited++)) - done - return 1 -} - -ensure_wg_interface() { - local node_id=$1 - local iface="wavekv-test${node_id}" - - # Check if interface exists, create if not - if ! ip link show "$iface" >/dev/null 2>&1; then - log_info "Creating WireGuard interface ${iface}..." - sudo ip link add "$iface" type wireguard || { - log_error "Failed to create WireGuard interface ${iface}" - return 1 - } - fi - return 0 -} - -start_node() { - local node_id=$1 - local config="${SCRIPT_DIR}/${RUN_DIR}/node${node_id}.toml" - local log_file="${LOG_DIR}/${CURRENT_TEST}_node${node_id}.log" - - # Calculate ports for this node - local admin_port=$((13000 + node_id * 10 + 6)) - local rpc_port=$((13000 + node_id * 10 + 2)) - - log_info "Starting node ${node_id}..." - - # Kill any existing test process for this node first (use absolute path to be precise) - pkill -9 -f "dstack-gateway -c ${config}" >/dev/null 2>&1 || true - pkill -9 -f "dstack-gateway.*${config}" >/dev/null 2>&1 || true - sleep 1 - - # Wait for ports to be free - if ! wait_for_port_free $admin_port; then - log_error "Port $admin_port still in use after waiting" - netstat -tlnp 2>/dev/null | grep ":${admin_port} " || true - return 1 - fi - if ! wait_for_port_free $rpc_port; then - log_error "Port $rpc_port still in use after waiting" - netstat -tlnp 2>/dev/null | grep ":${rpc_port} " || true - return 1 - fi - - # Ensure WireGuard interface exists before starting - if ! ensure_wg_interface "$node_id"; then - return 1 - fi - - mkdir -p "$RUN_DIR/wavekv_node${node_id}" - mkdir -p "$LOG_DIR" - (RUST_LOG=info "$GATEWAY_BIN" -c "$config" >"$log_file" 2>&1 &) - sleep 2 - - if pgrep -f "dstack-gateway.*${config}" >/dev/null; then - log_info "Node ${node_id} started successfully" - return 0 - else - log_error "Node ${node_id} failed to start" - cat "$log_file" - return 1 - fi -} - -stop_node() { - local node_id=$1 - local config="${SCRIPT_DIR}/${RUN_DIR}/node${node_id}.toml" - local admin_port=$((13000 + node_id * 10 + 6)) - - log_info "Stopping node ${node_id}..." - # Kill only the specific test process using absolute config path - pkill -9 -f "dstack-gateway -c ${config}" >/dev/null 2>&1 || true - pkill -9 -f "dstack-gateway.*${config}" >/dev/null 2>&1 || true - sleep 1 - - # Verify the port is free, otherwise force kill by PID - if ! wait_for_port_free $admin_port; then - log_warn "Node ${node_id} port still in use, forcing cleanup..." - # Find and kill the process holding the port - local pid=$(netstat -tlnp 2>/dev/null | grep ":${admin_port} " | awk '{print $7}' | cut -d'/' -f1) - if [[ -n "$pid" ]]; then - kill -9 "$pid" 2>/dev/null || true - sleep 1 - fi - fi - - # Reset terminal to fix any broken line endings - stty sane 2>/dev/null || true -} - -# Get WaveKV status via Admin.WaveKvStatus RPC -# Usage: get_status <admin_port> -get_status() { - local admin_port=$1 - curl -s -X POST "http://localhost:${admin_port}/prpc/Admin.WaveKvStatus" \ - -H "Content-Type: application/json" \ - -d '{}' 2>/dev/null -} - -get_n_keys() { - local admin_port=$1 - get_status "$admin_port" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['persistent']['n_keys'])" 2>/dev/null || echo "0" -} - -# Register CVM via debug port (no attestation required) -# Usage: debug_register_cvm <debug_port> <client_public_key> <app_id> <instance_id> -# Returns: JSON response -debug_register_cvm() { - local debug_port=$1 - local public_key=$2 - local app_id=${3:-"testapp"} - local instance_id=${4:-"testinstance"} - curl -s \ - -X POST "http://localhost:${debug_port}/prpc/RegisterCvm" \ - -H "Content-Type: application/json" \ - -d "{\"client_public_key\": \"$public_key\", \"app_id\": \"$app_id\", \"instance_id\": \"$instance_id\"}" 2>/dev/null -} - -# Check if debug service is available -# Usage: check_debug_service <debug_port> -check_debug_service() { - local debug_port=$1 - local response=$(curl -s -X POST "http://localhost:${debug_port}/prpc/Debug.Info" \ - -H "Content-Type: application/json" -d '{}' 2>/dev/null) - if echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); assert 'base_domain' in d" 2>/dev/null; then - return 0 - else - return 1 - fi -} - -# Verify register response is successful (has wg config, no error) -# Usage: verify_register_response <response> -verify_register_response() { - local response="$1" - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - if 'error' in d: - print(f'ERROR: {d[\"error\"]}', file=sys.stderr) - sys.exit(1) - assert 'wg' in d, 'missing wg config' - assert 'client_ip' in d['wg'], 'missing client_ip' - print(d['wg']['client_ip']) -except Exception as e: - print(f'ERROR: {e}', file=sys.stderr) - sys.exit(1) -" 2>/dev/null -} - -# Get sync data from debug port (peer_addrs, nodes, instances) -# Usage: debug_get_sync_data <debug_port> -# Returns: JSON response with my_node_id, peer_addrs, nodes, instances -debug_get_sync_data() { - local debug_port=$1 - curl -s -X POST "http://localhost:${debug_port}/prpc/Debug.GetSyncData" \ - -H "Content-Type: application/json" -d '{}' 2>/dev/null -} - -# Check if node has synced peer address from another node -# Usage: has_peer_addr <debug_port> <peer_node_id> -# Returns: 0 if peer address exists, 1 otherwise -has_peer_addr() { - local debug_port=$1 - local peer_node_id=$2 - local response=$(debug_get_sync_data "$debug_port") - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - peer_addrs = d.get('peer_addrs', []) - for pa in peer_addrs: - if pa.get('node_id') == $peer_node_id: - sys.exit(0) - sys.exit(1) -except Exception as e: - sys.exit(1) -" -} - -# Check if node has synced node info from another node -# Usage: has_node_info <debug_port> <peer_node_id> -# Returns: 0 if node info exists, 1 otherwise -has_node_info() { - local debug_port=$1 - local peer_node_id=$2 - local response=$(debug_get_sync_data "$debug_port") - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - nodes = d.get('nodes', []) - for n in nodes: - if n.get('node_id') == $peer_node_id: - sys.exit(0) - sys.exit(1) -except Exception as e: - sys.exit(1) -" -} - -# Get number of peer addresses from sync data -# Usage: get_n_peer_addrs <debug_port> -get_n_peer_addrs() { - local debug_port=$1 - local response=$(debug_get_sync_data "$debug_port") - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - print(len(d.get('peer_addrs', []))) -except Exception: - print(0) -" 2>/dev/null -} - -# Get number of node infos from sync data -# Usage: get_n_nodes <debug_port> -get_n_nodes() { - local debug_port=$1 - local response=$(debug_get_sync_data "$debug_port") - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - print(len(d.get('nodes', []))) -except Exception: - print(0) -" 2>/dev/null -} - -# Get number of instances from KvStore sync data -# Usage: get_n_instances <debug_port> -get_n_instances() { - local debug_port=$1 - local response=$(debug_get_sync_data "$debug_port") - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - print(len(d.get('instances', []))) -except Exception: - print(0) -" 2>/dev/null -} - -# Get Proxy State from debug port (in-memory state) -# Usage: debug_get_proxy_state <debug_port> -# Returns: JSON response with instances and allocated_addresses -debug_get_proxy_state() { - local debug_port=$1 - curl -s -X POST "http://localhost:${debug_port}/prpc/GetProxyState" \ - -H "Content-Type: application/json" -d '{}' 2>/dev/null -} - -# Get number of instances from ProxyState (in-memory) -# Usage: get_n_proxy_state_instances <debug_port> -get_n_proxy_state_instances() { - local debug_port=$1 - local response=$(debug_get_proxy_state "$debug_port") - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - print(len(d.get('instances', []))) -except Exception: - print(0) -" 2>/dev/null -} - -# Check KvStore and ProxyState instance consistency -# Usage: check_instance_consistency <debug_port> -# Returns: 0 if consistent, 1 otherwise -check_instance_consistency() { - local debug_port=$1 - local kvstore_instances=$(get_n_instances "$debug_port") - local proxystate_instances=$(get_n_proxy_state_instances "$debug_port") - - if [[ "$kvstore_instances" -eq "$proxystate_instances" ]]; then - return 0 - else - log_error "Instance count mismatch: KvStore=$kvstore_instances, ProxyState=$proxystate_instances" - return 1 - fi -} - -# ============================================================================= -# Test 1: Single node persistence -# ============================================================================= -test_persistence() { - log_info "========== Test 1: Persistence ==========" - cleanup - - generate_config 1 - - # Start node and let it write some data - start_node 1 - - local admin_port=13016 - local initial_keys=$(get_n_keys $admin_port) - log_info "Initial keys: $initial_keys" - - # The gateway auto-writes some data (peer_addr, etc) - sleep 2 - local keys_after_write=$(get_n_keys $admin_port) - log_info "Keys after startup: $keys_after_write" - - # Stop and restart - stop_node 1 - log_info "Restarting node 1..." - start_node 1 - - local keys_after_restart=$(get_n_keys $admin_port) - log_info "Keys after restart: $keys_after_restart" - - if [[ "$keys_after_restart" -ge "$keys_after_write" ]]; then - log_info "Persistence test PASSED" - return 0 - else - log_error "Persistence test FAILED: expected >= $keys_after_write keys, got $keys_after_restart" - return 1 - fi -} - -# ============================================================================= -# Test 2: Multi-node sync -# ============================================================================= -test_multi_node_sync() { - log_info "========== Test 2: Multi-node Sync ==========" - cleanup - - # Clean up all state files to ensure fresh start - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" "$RUN_DIR/wavekv_node3" - rm -f "$RUN_DIR/gateway-state-node1.json" "$RUN_DIR/gateway-state-node2.json" "$RUN_DIR/gateway-state-node3.json" - - generate_config 1 - generate_config 2 - - start_node 1 - start_node 2 - - # Register peers so nodes can discover each other - setup_peers 1 2 - - local debug_port1=13015 - local debug_port2=13025 - - # Wait for sync - log_info "Waiting for nodes to sync..." - sleep 10 - - # Use debug RPC to check actual synced data - local peer_addrs1=$(get_n_peer_addrs $debug_port1) - local peer_addrs2=$(get_n_peer_addrs $debug_port2) - local nodes1=$(get_n_nodes $debug_port1) - local nodes2=$(get_n_nodes $debug_port2) - - log_info "Node 1: peer_addrs=$peer_addrs1, nodes=$nodes1" - log_info "Node 2: peer_addrs=$peer_addrs2, nodes=$nodes2" - - # For true sync, each node should have: - # - At least 2 peer addresses (both nodes' addresses) - # - At least 2 node infos (both nodes' info) - local sync_ok=true - - if ! has_peer_addr $debug_port1 2; then - log_error "Node 1 missing peer_addr for node 2" - sync_ok=false - fi - if ! has_peer_addr $debug_port2 1; then - log_error "Node 2 missing peer_addr for node 1" - sync_ok=false - fi - if ! has_node_info $debug_port1 2; then - log_error "Node 1 missing node_info for node 2" - sync_ok=false - fi - if ! has_node_info $debug_port2 1; then - log_error "Node 2 missing node_info for node 1" - sync_ok=false - fi - - if [[ "$sync_ok" == "true" ]]; then - log_info "Multi-node sync test PASSED" - return 0 - else - log_error "Multi-node sync test FAILED: nodes did not sync peer data" - log_info "Sync data from node 1: $(debug_get_sync_data $debug_port1)" - log_info "Sync data from node 2: $(debug_get_sync_data $debug_port2)" - return 1 - fi -} - -# ============================================================================= -# Test 3: Node recovery after disconnect -# ============================================================================= -test_node_recovery() { - log_info "========== Test 3: Node Recovery ==========" - cleanup - - # Clean up all state files to ensure fresh start - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" - rm -f "$RUN_DIR/gateway-state-node1.json" "$RUN_DIR/gateway-state-node2.json" - - generate_config 1 - generate_config 2 - - start_node 1 - start_node 2 - - # Register peers so nodes can discover each other - setup_peers 1 2 - - local debug_port1=13015 - local debug_port2=13025 - - # Wait for initial sync - sleep 5 - - # Stop node 2 - log_info "Stopping node 2 to simulate disconnect..." - stop_node 2 - - # Wait and let node 1 continue - sleep 3 - - # Check node 1 has its own data - local peer_addrs1_before=$(get_n_peer_addrs $debug_port1) - log_info "Node 1 peer_addrs before node 2 restart: $peer_addrs1_before" - - # Restart node 2 - log_info "Restarting node 2..." - start_node 2 - - # Re-register peers after restart - setup_peers 1 2 - - # Wait for sync - sleep 10 - - # After recovery, node 2 should have synced node 1's data - local sync_ok=true - - if ! has_peer_addr $debug_port2 1; then - log_error "Node 2 missing peer_addr for node 1 after recovery" - sync_ok=false - fi - if ! has_node_info $debug_port2 1; then - log_error "Node 2 missing node_info for node 1 after recovery" - sync_ok=false - fi - - if [[ "$sync_ok" == "true" ]]; then - log_info "Node recovery test PASSED" - return 0 - else - log_error "Node recovery test FAILED: node 2 did not sync data from node 1" - log_info "Sync data from node 2: $(debug_get_sync_data $debug_port2)" - return 1 - fi -} - -# ============================================================================= -# Test 4: Status endpoint structure (Admin.WaveKvStatus RPC) -# ============================================================================= -test_status_endpoint() { - log_info "========== Test 4: Status Endpoint ==========" - cleanup - - generate_config 1 - start_node 1 - - local admin_port=13016 - local status=$(get_status $admin_port) - - # Verify all expected fields exist - local checks_passed=0 - local total_checks=6 - - echo "$status" | python3 -c " -import sys, json -d = json.load(sys.stdin) -assert d['enabled'] == True, 'enabled should be True' -assert 'persistent' in d, 'missing persistent' -assert 'ephemeral' in d, 'missing ephemeral' -assert d['persistent']['wal_enabled'] == True, 'persistent wal should be enabled' -assert d['ephemeral']['wal_enabled'] == False, 'ephemeral wal should be disabled' -assert 'peers' in d['persistent'], 'missing peers in persistent' -print('All status checks passed') -" && checks_passed=1 - - if [[ $checks_passed -eq 1 ]]; then - log_info "Status endpoint test PASSED" - return 0 - else - log_error "Status endpoint test FAILED" - log_info "Status response: $status" - return 1 - fi -} - -# ============================================================================= -# Test 5: Cross-node data sync verification (KvStore + ProxyState) -# ============================================================================= -test_cross_node_data_sync() { - log_info "========== Test 5: Cross-node Data Sync ==========" - cleanup - - generate_config 1 - generate_config 2 - - start_node 1 - start_node 2 - - # Register peers so nodes can discover each other - setup_peers 1 2 - - local debug_port1=13015 - local debug_port2=13025 - - # Wait for initial connection - sleep 5 - - # Verify debug service is available - if ! check_debug_service $debug_port1; then - log_error "Debug service not available on node 1" - return 1 - fi - - # Register a client on node 1 via debug port - log_info "Registering client on node 1 via debug port..." - local register_response=$(debug_register_cvm $debug_port1 "testkey12345678901234567890123456789012345=" "app1" "inst1") - log_info "Register response: $register_response" - - # Verify registration succeeded - local client_ip=$(verify_register_response "$register_response") - if [[ -z "$client_ip" ]]; then - log_error "Registration failed" - return 1 - fi - log_info "Registered client with IP: $client_ip" - - # Wait for sync (need at least 3 sync intervals of 5s for data to propagate) - log_info "Waiting for sync..." - sleep 20 - - # Check KvStore instance count on both nodes - local kv_instances1=$(get_n_instances $debug_port1) - local kv_instances2=$(get_n_instances $debug_port2) - - # Check ProxyState instance count on both nodes - local ps_instances1=$(get_n_proxy_state_instances $debug_port1) - local ps_instances2=$(get_n_proxy_state_instances $debug_port2) - - log_info "Node 1: KvStore=$kv_instances1, ProxyState=$ps_instances1" - log_info "Node 2: KvStore=$kv_instances2, ProxyState=$ps_instances2" - - local test_passed=true - - # Verify KvStore sync - if [[ "$kv_instances1" -lt 1 ]] || [[ "$kv_instances2" -lt 1 ]]; then - log_error "KvStore sync failed: kv_instances1=$kv_instances1, kv_instances2=$kv_instances2" - test_passed=false - fi - - # Verify ProxyState sync (node 2 should have loaded instance from KvStore) - if [[ "$ps_instances1" -lt 1 ]] || [[ "$ps_instances2" -lt 1 ]]; then - log_error "ProxyState sync failed: ps_instances1=$ps_instances1, ps_instances2=$ps_instances2" - test_passed=false - fi - - # Verify consistency on each node - if [[ "$kv_instances1" -ne "$ps_instances1" ]]; then - log_error "Node 1 inconsistent: KvStore=$kv_instances1, ProxyState=$ps_instances1" - test_passed=false - fi - if [[ "$kv_instances2" -ne "$ps_instances2" ]]; then - log_error "Node 2 inconsistent: KvStore=$kv_instances2, ProxyState=$ps_instances2" - test_passed=false - fi - - if [[ "$test_passed" == "true" ]]; then - log_info "Cross-node data sync test PASSED (KvStore and ProxyState consistent)" - return 0 - else - log_info "KvStore from node 1: $(debug_get_sync_data $debug_port1)" - log_info "KvStore from node 2: $(debug_get_sync_data $debug_port2)" - log_info "ProxyState from node 1: $(debug_get_proxy_state $debug_port1)" - log_info "ProxyState from node 2: $(debug_get_proxy_state $debug_port2)" - return 1 - fi -} - -# ============================================================================= -# Test 6: prpc DebugRegisterCvm endpoint (on separate debug port) -# ============================================================================= -test_prpc_register() { - log_info "========== Test 6: prpc DebugRegisterCvm ==========" - cleanup - - generate_config 1 - start_node 1 - - local debug_port=13015 - - # Verify debug service is available first - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - log_info "Debug service is available" - - # Register via debug port - local register_response=$(debug_register_cvm $debug_port "prpctest12345678901234567890123456789012=" "deadbeef" "cafebabe") - log_info "Register response: $register_response" - - # Verify registration succeeded - local client_ip=$(verify_register_response "$register_response") - if [[ -z "$client_ip" ]]; then - log_error "prpc DebugRegisterCvm test FAILED" - return 1 - fi - - log_info "DebugRegisterCvm success: client_ip=$client_ip" - log_info "prpc DebugRegisterCvm test PASSED" - return 0 -} - -# ============================================================================= -# Test 7: prpc Info endpoint -# ============================================================================= -test_prpc_info() { - log_info "========== Test 7: prpc Info ==========" - cleanup - - generate_config 1 - start_node 1 - - local port=13012 - - # Call Info via prpc - # Note: trim: "Tproxy." removes "Tproxy.Gateway." prefix, so endpoint is just /prpc/Info - local info_response=$(curl -sk --cacert "$CA_CERT" \ - -X POST "https://localhost:${port}/prpc/Info" \ - -H "Content-Type: application/json" \ - -d '{}' 2>/dev/null) - - log_info "Info response: $info_response" - - # Verify response has expected fields and no error - echo "$info_response" | python3 -c " -import sys, json -d = json.load(sys.stdin) -if 'error' in d: - print(f'ERROR: {d[\"error\"]}', file=sys.stderr) - sys.exit(1) -assert 'base_domain' in d, 'missing base_domain' -assert 'external_port' in d, 'missing external_port' -print('prpc Info check passed') -" && { - log_info "prpc Info test PASSED" - return 0 - } || { - log_error "prpc Info test FAILED" - return 1 - } -} - -# ============================================================================= -# Test 8: Client registration and data persistence -# ============================================================================= -test_client_registration_persistence() { - log_info "========== Test 8: Client Registration Persistence ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" - - generate_config 1 - start_node 1 - - local debug_port=13015 - local admin_port=13016 - - # Verify debug service is available - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - - # Register a client via debug port - log_info "Registering client..." - local register_response=$(debug_register_cvm $debug_port "persisttest1234567890123456789012345678901=" "persist_app" "persist_inst") - log_info "Register response: $register_response" - - # Verify registration succeeded - local client_ip=$(verify_register_response "$register_response") - if [[ -z "$client_ip" ]]; then - log_error "Registration failed" - return 1 - fi - - # Get initial key count - local keys_before=$(get_n_keys $admin_port) - log_info "Keys before restart: $keys_before" - - # Restart node - stop_node 1 - start_node 1 - - # Check keys after restart - local keys_after=$(get_n_keys $admin_port) - log_info "Keys after restart: $keys_after" - - if [[ "$keys_after" -ge "$keys_before" ]] && [[ "$keys_before" -gt 2 ]]; then - log_info "Client registration persistence test PASSED" - return 0 - else - log_error "Client registration persistence test FAILED: keys_before=$keys_before, keys_after=$keys_after" - return 1 - fi -} - -# ============================================================================= -# Test 9: Stress test - multiple writes -# ============================================================================= -test_stress_writes() { - log_info "========== Test 9: Stress Test ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" - - generate_config 1 - start_node 1 - - local debug_port=13015 - local admin_port=13016 - local num_clients=10 - local success_count=0 - - # Verify debug service is available - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - - log_info "Registering $num_clients clients via debug port..." - for i in $(seq 1 $num_clients); do - local key=$(printf "stresstest%02d12345678901234567890123456=" "$i") - local app_id=$(printf "stressapp%02d" "$i") - local inst_id=$(printf "stressinst%02d" "$i") - local response=$(debug_register_cvm $debug_port "$key" "$app_id" "$inst_id") - if verify_register_response "$response" >/dev/null 2>&1; then - ((success_count++)) - fi - done - - log_info "Successfully registered $success_count/$num_clients clients" - - sleep 2 - - local keys_after=$(get_n_keys $admin_port) - log_info "Keys after stress test: $keys_after" - - # We expect successful registrations to create keys - if [[ "$success_count" -eq "$num_clients" ]] && [[ "$keys_after" -gt 2 ]]; then - log_info "Stress test PASSED" - return 0 - else - log_error "Stress test FAILED: success_count=$success_count, keys_after=$keys_after" - return 1 - fi -} - -# ============================================================================= -# Test 10: Network partition simulation (KvStore + ProxyState consistency) -# ============================================================================= -test_network_partition() { - log_info "========== Test 10: Network Partition Recovery ==========" - cleanup - - # Clean up all state files to ensure fresh start - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" - rm -f "$RUN_DIR/gateway-state-node1.json" "$RUN_DIR/gateway-state-node2.json" - - generate_config 1 - generate_config 2 - - start_node 1 - start_node 2 - - # Register peers so nodes can discover each other - setup_peers 1 2 - - local debug_port1=13015 - local debug_port2=13025 - - # Let them sync initially - sleep 5 - - # Verify debug service is available - if ! check_debug_service $debug_port1; then - log_error "Debug service not available on node 1" - return 1 - fi - - # Stop node 2 (simulate partition) - log_info "Simulating network partition - stopping node 2..." - stop_node 2 - - # Register clients on node 1 while node 2 is down - log_info "Registering clients on node 1 during partition..." - local success_count=0 - for i in $(seq 1 3); do - local key=$(printf "partition%02d123456789012345678901234567=" "$i") - local response=$(debug_register_cvm $debug_port1 "$key" "partition_app$i" "partition_inst$i") - if verify_register_response "$response" >/dev/null 2>&1; then - ((success_count++)) - fi - done - log_info "Registered $success_count/3 clients during partition" - - local kv1_during=$(get_n_instances $debug_port1) - local ps1_during=$(get_n_proxy_state_instances $debug_port1) - log_info "Node 1 during partition: KvStore=$kv1_during, ProxyState=$ps1_during" - - # Restore node 2 - log_info "Healing partition - restarting node 2..." - start_node 2 - - # Re-register peers after restart - setup_peers 1 2 - - # Wait for sync - sleep 15 - - # Check KvStore and ProxyState on both nodes after recovery - local kv1_after=$(get_n_instances $debug_port1) - local kv2_after=$(get_n_instances $debug_port2) - local ps1_after=$(get_n_proxy_state_instances $debug_port1) - local ps2_after=$(get_n_proxy_state_instances $debug_port2) - - log_info "Node 1 after recovery: KvStore=$kv1_after, ProxyState=$ps1_after" - log_info "Node 2 after recovery: KvStore=$kv2_after, ProxyState=$ps2_after" - - local test_passed=true - - # Verify basic sync - if [[ "$success_count" -ne 3 ]] || [[ "$kv1_during" -lt 3 ]]; then - log_error "Registration or KvStore write failed during partition" - test_passed=false - fi - - # Verify node 2 synced KvStore - if [[ "$kv2_after" -lt "$kv1_during" ]]; then - log_error "Node 2 KvStore sync failed: kv2_after=$kv2_after, expected >= $kv1_during" - test_passed=false - fi - - # Verify node 2 ProxyState sync - if [[ "$ps2_after" -lt "$kv1_during" ]]; then - log_error "Node 2 ProxyState sync failed: ps2_after=$ps2_after, expected >= $kv1_during" - test_passed=false - fi - - # Verify consistency on each node - if [[ "$kv1_after" -ne "$ps1_after" ]]; then - log_error "Node 1 inconsistent: KvStore=$kv1_after, ProxyState=$ps1_after" - test_passed=false - fi - if [[ "$kv2_after" -ne "$ps2_after" ]]; then - log_error "Node 2 inconsistent: KvStore=$kv2_after, ProxyState=$ps2_after" - test_passed=false - fi - - if [[ "$test_passed" == "true" ]]; then - log_info "Network partition recovery test PASSED (KvStore and ProxyState consistent)" - return 0 - else - log_info "KvStore from node 2: $(debug_get_sync_data $debug_port2)" - log_info "ProxyState from node 2: $(debug_get_proxy_state $debug_port2)" - return 1 - fi -} - -# ============================================================================= -# Test 11: Three-node cluster (KvStore + ProxyState consistency) -# ============================================================================= -test_three_node_cluster() { - log_info "========== Test 11: Three-node Cluster ==========" - cleanup - - # Clean up all state files to ensure fresh start - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" "$RUN_DIR/wavekv_node3" - rm -f "$RUN_DIR/gateway-state-node1.json" "$RUN_DIR/gateway-state-node2.json" "$RUN_DIR/gateway-state-node3.json" - - generate_config 1 - generate_config 2 - generate_config 3 - - start_node 1 - start_node 2 - start_node 3 - - # Register peers so all nodes can discover each other - setup_peers 1 2 3 - - local debug_port1=13015 - local debug_port2=13025 - local debug_port3=13035 - - # Wait for cluster to form - sleep 10 - - # Verify debug service is available - if ! check_debug_service $debug_port1; then - log_error "Debug service not available on node 1" - return 1 - fi - - # Register client on node 1 - log_info "Registering client on node 1..." - local response=$(debug_register_cvm $debug_port1 "threenode12345678901234567890123456789=" "threenode_app" "threenode_inst") - local client_ip=$(verify_register_response "$response") - if [[ -z "$client_ip" ]]; then - log_error "Registration failed" - return 1 - fi - log_info "Registered client with IP: $client_ip" - - # Wait for sync across all nodes (need at least 2 sync intervals of 5s) - sleep 20 - - # Check KvStore instances on all three nodes - local kv1=$(get_n_instances $debug_port1) - local kv2=$(get_n_instances $debug_port2) - local kv3=$(get_n_instances $debug_port3) - - # Check ProxyState instances on all three nodes - local ps1=$(get_n_proxy_state_instances $debug_port1) - local ps2=$(get_n_proxy_state_instances $debug_port2) - local ps3=$(get_n_proxy_state_instances $debug_port3) - - log_info "Node 1: KvStore=$kv1, ProxyState=$ps1" - log_info "Node 2: KvStore=$kv2, ProxyState=$ps2" - log_info "Node 3: KvStore=$kv3, ProxyState=$ps3" - - local test_passed=true - - # Verify KvStore sync on all nodes - if [[ "$kv1" -lt 1 ]] || [[ "$kv2" -lt 1 ]] || [[ "$kv3" -lt 1 ]]; then - log_error "KvStore sync failed: kv1=$kv1, kv2=$kv2, kv3=$kv3" - test_passed=false - fi - - # Verify ProxyState sync on all nodes - if [[ "$ps1" -lt 1 ]] || [[ "$ps2" -lt 1 ]] || [[ "$ps3" -lt 1 ]]; then - log_error "ProxyState sync failed: ps1=$ps1, ps2=$ps2, ps3=$ps3" - test_passed=false - fi - - # Verify consistency on each node - if [[ "$kv1" -ne "$ps1" ]] || [[ "$kv2" -ne "$ps2" ]] || [[ "$kv3" -ne "$ps3" ]]; then - log_error "Inconsistency detected between KvStore and ProxyState" - test_passed=false - fi - - if [[ "$test_passed" == "true" ]]; then - log_info "Three-node cluster test PASSED (KvStore and ProxyState consistent)" - return 0 - else - log_info "KvStore from node 1: $(debug_get_sync_data $debug_port1)" - log_info "KvStore from node 2: $(debug_get_sync_data $debug_port2)" - log_info "KvStore from node 3: $(debug_get_sync_data $debug_port3)" - log_info "ProxyState from node 1: $(debug_get_proxy_state $debug_port1)" - log_info "ProxyState from node 2: $(debug_get_proxy_state $debug_port2)" - log_info "ProxyState from node 3: $(debug_get_proxy_state $debug_port3)" - return 1 - fi -} - -# ============================================================================= -# Test 12: WAL file integrity -# ============================================================================= -test_wal_integrity() { - log_info "========== Test 12: WAL File Integrity ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" - - generate_config 1 - start_node 1 - - local debug_port=13015 - local success_count=0 - - # Verify debug service is available - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - - # Register some clients via debug port - for i in $(seq 1 5); do - local key=$(printf "waltest%02d1234567890123456789012345678901=" "$i") - local response=$(debug_register_cvm $debug_port "$key" "wal_app$i" "wal_inst$i") - if verify_register_response "$response" >/dev/null 2>&1; then - ((success_count++)) - fi - done - log_info "Registered $success_count/5 clients" - - if [[ "$success_count" -ne 5 ]]; then - log_error "Failed to register all clients" - return 1 - fi - - sleep 2 - stop_node 1 - - # Check WAL file exists and has content - local wal_file="$RUN_DIR/wavekv_node1/node_1.wal" - if [[ -f "$wal_file" ]]; then - local wal_size=$(stat -c%s "$wal_file" 2>/dev/null || stat -f%z "$wal_file" 2>/dev/null) - log_info "WAL file size: $wal_size bytes" - - if [[ "$wal_size" -gt 100 ]]; then - log_info "WAL file integrity test PASSED" - return 0 - else - log_error "WAL file integrity test FAILED: WAL file too small ($wal_size bytes)" - return 1 - fi - else - log_error "WAL file not found: $wal_file" - return 1 - fi -} - -# ============================================================================= -# Test 13: Three-node cluster with bootnode (no dynamic peer setup) -# ============================================================================= -test_three_node_bootnode() { - log_info "========== Test 13: Three-node Cluster with Bootnode ==========" - cleanup - - # Clean up all state files to ensure fresh start - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" "$RUN_DIR/wavekv_node3" - rm -f "$RUN_DIR/gateway-state-node1.json" "$RUN_DIR/gateway-state-node2.json" "$RUN_DIR/gateway-state-node3.json" - - # Node 1 is the bootnode (no bootnode config) - # Node 2 and 3 use node 1 as bootnode - local bootnode_url="https://localhost:13012" - - generate_config 1 "" - generate_config 2 "$bootnode_url" - generate_config 3 "$bootnode_url" - - # Start node 1 first (bootnode) - start_node 1 - sleep 2 - - # Start node 2 and 3, they will discover each other via bootnode - start_node 2 - start_node 3 - - local debug_port1=13015 - local debug_port2=13025 - local debug_port3=13035 - - # Wait for cluster to form via bootnode discovery - log_info "Waiting for nodes to discover each other via bootnode..." - sleep 15 - - # Verify debug service is available on all nodes - for port in $debug_port1 $debug_port2 $debug_port3; do - if ! check_debug_service $port; then - log_error "Debug service not available on port $port" - return 1 - fi - done - - # Check peer discovery - each node should know about the others - local peer_addrs1=$(get_n_peer_addrs $debug_port1) - local peer_addrs2=$(get_n_peer_addrs $debug_port2) - local peer_addrs3=$(get_n_peer_addrs $debug_port3) - - log_info "Peer addresses: node1=$peer_addrs1, node2=$peer_addrs2, node3=$peer_addrs3" - - # Register client on node 2 (not the bootnode) - log_info "Registering client on node 2..." - local response=$(debug_register_cvm $debug_port2 "bootnode12345678901234567890123456789=" "bootnode_app" "bootnode_inst") - local client_ip=$(verify_register_response "$response") - if [[ -z "$client_ip" ]]; then - log_error "Registration failed" - return 1 - fi - log_info "Registered client with IP: $client_ip" - - # Wait for sync across all nodes - sleep 20 - - # Check KvStore instances on all three nodes - local kv1=$(get_n_instances $debug_port1) - local kv2=$(get_n_instances $debug_port2) - local kv3=$(get_n_instances $debug_port3) - - # Check ProxyState instances on all three nodes - local ps1=$(get_n_proxy_state_instances $debug_port1) - local ps2=$(get_n_proxy_state_instances $debug_port2) - local ps3=$(get_n_proxy_state_instances $debug_port3) - - log_info "Node 1 (bootnode): KvStore=$kv1, ProxyState=$ps1" - log_info "Node 2: KvStore=$kv2, ProxyState=$ps2" - log_info "Node 3: KvStore=$kv3, ProxyState=$ps3" - - local test_passed=true - - # Verify peer discovery worked (each node should have at least 2 peer addresses) - if [[ "$peer_addrs1" -lt 2 ]] || [[ "$peer_addrs2" -lt 2 ]] || [[ "$peer_addrs3" -lt 2 ]]; then - log_error "Peer discovery via bootnode failed: peer_addrs1=$peer_addrs1, peer_addrs2=$peer_addrs2, peer_addrs3=$peer_addrs3" - test_passed=false - fi - - # Verify KvStore sync on all nodes - if [[ "$kv1" -lt 1 ]] || [[ "$kv2" -lt 1 ]] || [[ "$kv3" -lt 1 ]]; then - log_error "KvStore sync failed: kv1=$kv1, kv2=$kv2, kv3=$kv3" - test_passed=false - fi - - # Verify ProxyState sync on all nodes - if [[ "$ps1" -lt 1 ]] || [[ "$ps2" -lt 1 ]] || [[ "$ps3" -lt 1 ]]; then - log_error "ProxyState sync failed: ps1=$ps1, ps2=$ps2, ps3=$ps3" - test_passed=false - fi - - # Verify consistency on each node - if [[ "$kv1" -ne "$ps1" ]] || [[ "$kv2" -ne "$ps2" ]] || [[ "$kv3" -ne "$ps3" ]]; then - log_error "Inconsistency detected between KvStore and ProxyState" - test_passed=false - fi - - if [[ "$test_passed" == "true" ]]; then - log_info "Three-node bootnode cluster test PASSED" - return 0 - else - log_info "Sync data from node 1: $(debug_get_sync_data $debug_port1)" - log_info "Sync data from node 2: $(debug_get_sync_data $debug_port2)" - log_info "Sync data from node 3: $(debug_get_sync_data $debug_port3)" - return 1 - fi -} - -# ============================================================================= -# Test 14: Node ID reuse rejection -# ============================================================================= -test_node_id_reuse_rejected() { - log_info "========== Test 14: Node ID Reuse Rejected ==========" - cleanup - - # Clean up all state files to ensure fresh start - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" - rm -f "$RUN_DIR/gateway-state-node1.json" "$RUN_DIR/gateway-state-node2.json" - - # Start node 1 and node 2, let them sync - generate_config 1 - generate_config 2 - - start_node 1 - start_node 2 - - # Register peers so nodes can discover each other - setup_peers 1 2 - - local debug_port1=13015 - local debug_port2=13025 - local admin_port1=13016 - - # Wait for initial sync - log_info "Waiting for initial sync between node 1 and node 2..." - sleep 10 - - # Verify both nodes have synced - if ! has_peer_addr $debug_port1 2; then - log_error "Node 1 missing peer_addr for node 2" - return 1 - fi - if ! has_peer_addr $debug_port2 1; then - log_error "Node 2 missing peer_addr for node 1" - return 1 - fi - log_info "Initial sync completed successfully" - - # Get initial key count on node 1 - local keys_before=$(get_n_keys $admin_port1) - log_info "Keys on node 1 before node 2 restart: $keys_before" - - # Stop node 2 and delete its data (simulating a fresh node trying to reuse the ID) - log_info "Stopping node 2 and deleting its data..." - stop_node 2 - rm -rf "$RUN_DIR/wavekv_node2" - rm -f "$RUN_DIR/gateway-state-node2.json" - - # Restart node 2 - it will have a new UUID but same node_id - log_info "Restarting node 2 with fresh data (new UUID, same node_id)..." - start_node 2 - - # Re-register peers - setup_peers 1 2 - - # Wait for sync attempt - sleep 15 - - # Check node 2's log for UUID mismatch error - local log_file="${LOG_DIR}/${CURRENT_TEST}_node2.log" - if grep -q "UUID mismatch" "$log_file" 2>/dev/null; then - log_info "Found UUID mismatch error in node 2 log (expected)" - else - log_warn "UUID mismatch error not found in log (may still be rejected)" - fi - - # Node 1 should have rejected sync from new node 2 - # Check if node 1's data is still intact (keys should not decrease) - local keys_after=$(get_n_keys $admin_port1) - log_info "Keys on node 1 after node 2 restart: $keys_after" - - # The new node 2 should NOT have received data from node 1 - # because node 1 should reject sync due to UUID mismatch - local kv2=$(get_n_instances $debug_port2) - log_info "Node 2 instances after restart: $kv2" - - # Verify node 1's data is intact - if [[ "$keys_after" -lt "$keys_before" ]]; then - log_error "Node 1 lost data after node 2 restart with reused ID" - return 1 - fi - - # The test passes if: - # 1. Node 1's data is intact - # 2. Either UUID mismatch was logged OR node 2 didn't get full sync - log_info "Node ID reuse rejection test PASSED" - return 0 -} - -# ============================================================================= -# Test 15: Periodic persistence -# ============================================================================= -test_periodic_persistence() { - log_info "========== Test 15: Periodic Persistence ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" - - generate_config 1 - start_node 1 - - local debug_port=13015 - local admin_port=13016 - - # Verify debug service is available - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - - # Register some clients to create data - log_info "Registering clients to create data..." - local success_count=0 - for i in $(seq 1 3); do - local key=$(printf "persist%02d123456789012345678901234567890=" "$i") - local response=$(debug_register_cvm $debug_port "$key" "persist_app$i" "persist_inst$i") - if verify_register_response "$response" >/dev/null 2>&1; then - ((success_count++)) - fi - done - log_info "Registered $success_count/3 clients" - - if [[ "$success_count" -ne 3 ]]; then - log_error "Failed to register all clients" - return 1 - fi - - # Get initial key count - local keys_before=$(get_n_keys $admin_port) - log_info "Keys before waiting for persist: $keys_before" - - # Wait for periodic persistence (persist_interval is 5s in test config) - log_info "Waiting for periodic persistence (8s)..." - sleep 8 - - # Check log for periodic persist message - local log_file="${LOG_DIR}/${CURRENT_TEST}_node1.log" - if grep -q "periodic persist completed" "$log_file" 2>/dev/null; then - log_info "Found periodic persist message in log" - else - log_error "Periodic persist message not found in log - test FAILED" - return 1 - fi - - # Stop node - stop_node 1 - - # Check WAL file exists and has content - local wal_file="$RUN_DIR/wavekv_node1/node_1.wal" - if [[ ! -f "$wal_file" ]]; then - log_error "WAL file not found: $wal_file" - return 1 - fi - - local wal_size=$(stat -c%s "$wal_file" 2>/dev/null || stat -f%z "$wal_file" 2>/dev/null) - log_info "WAL file size after periodic persist: $wal_size bytes" - - # Restart node and verify data is recovered - log_info "Restarting node to verify persistence..." - start_node 1 - - local keys_after=$(get_n_keys $admin_port) - log_info "Keys after restart: $keys_after" - - if [[ "$keys_after" -ge "$keys_before" ]]; then - log_info "Periodic persistence test PASSED" - return 0 - else - log_error "Periodic persistence test FAILED: keys_before=$keys_before, keys_after=$keys_after" - return 1 - fi -} - -# ============================================================================= -# Admin RPC helper functions -# ============================================================================= - -# Call Admin.SetNodeUrl RPC -# Usage: admin_set_node_url <admin_port> <node_id> <url> -admin_set_node_url() { - local admin_port=$1 - local node_id=$2 - local url=$3 - curl -s -X POST "http://localhost:${admin_port}/prpc/Admin.SetNodeUrl" \ - -H "Content-Type: application/json" \ - -d "{\"id\": $node_id, \"url\": \"$url\"}" 2>/dev/null -} - -# Register peers between nodes via Admin RPC -# This is needed since we removed peer_node_ids/peer_urls from config -# Usage: setup_peers <node_ids...> -# Example: setup_peers 1 2 3 # Sets up peers between nodes 1, 2, and 3 -setup_peers() { - local node_ids=("$@") - - for src_node in "${node_ids[@]}"; do - local src_admin_port=$((13000 + src_node * 10 + 6)) - - for dst_node in "${node_ids[@]}"; do - if [[ "$src_node" != "$dst_node" ]]; then - local dst_rpc_port=$((13000 + dst_node * 10 + 2)) - local dst_url="https://localhost:${dst_rpc_port}" - admin_set_node_url "$src_admin_port" "$dst_node" "$dst_url" - fi - done - done - - # Wait for peers to be registered - sleep 1 -} - -# Call Admin.SetNodeStatus RPC -# Usage: admin_set_node_status <admin_port> <node_id> <status> -# status: "up" or "down" -admin_set_node_status() { - local admin_port=$1 - local node_id=$2 - local status=$3 - curl -s -X POST "http://localhost:${admin_port}/prpc/Admin.SetNodeStatus" \ - -H "Content-Type: application/json" \ - -d "{\"id\": $node_id, \"status\": \"$status\"}" 2>/dev/null -} - -# Call Admin.Status RPC to get all nodes -# Usage: admin_get_status <admin_port> -admin_get_status() { - local admin_port=$1 - curl -s -X POST "http://localhost:${admin_port}/prpc/Admin.Status" \ - -H "Content-Type: application/json" \ - -d '{}' 2>/dev/null -} - -# Get peer URL from sync data -# Usage: get_peer_url <debug_port> <node_id> -get_peer_url_from_sync() { - local debug_port=$1 - local node_id=$2 - local response=$(debug_get_sync_data "$debug_port") - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - for pa in d.get('peer_addrs', []): - if pa.get('node_id') == $node_id: - print(pa.get('url', '')) - sys.exit(0) - print('') -except Exception: - print('') -" 2>/dev/null -} - -# ============================================================================= -# Test 16: Admin.SetNodeUrl RPC -# ============================================================================= -test_admin_set_node_url() { - log_info "========== Test 16: Admin.SetNodeUrl RPC ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" - - generate_config 1 - start_node 1 - - local admin_port=13016 - local debug_port=13015 - - # Verify debug service is available - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - - # Set URL for a new node (node 2) via Admin RPC - local new_url="https://new-node2.example.com:8011" - log_info "Setting node 2 URL via Admin.SetNodeUrl..." - local response=$(admin_set_node_url $admin_port 2 "$new_url") - log_info "SetNodeUrl response: $response" - - # Check if the response contains an error - if echo "$response" | grep -q '"error"'; then - log_error "SetNodeUrl returned error: $response" - return 1 - fi - - # Wait for data to be written - sleep 2 - - # Verify the URL was stored in KvStore - local stored_url=$(get_peer_url_from_sync $debug_port 2) - log_info "Stored URL for node 2: $stored_url" - - if [[ "$stored_url" == "$new_url" ]]; then - log_info "Admin.SetNodeUrl test PASSED" - return 0 - else - log_error "Admin.SetNodeUrl test FAILED: expected '$new_url', got '$stored_url'" - log_info "Sync data: $(debug_get_sync_data $debug_port)" - return 1 - fi -} - -# ============================================================================= -# Test 17: Admin.SetNodeStatus RPC -# ============================================================================= -test_admin_set_node_status() { - log_info "========== Test 17: Admin.SetNodeStatus RPC ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" - - generate_config 1 - start_node 1 - - local admin_port=13016 - local debug_port=13015 - - # Verify debug service is available - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - - # First set a URL for node 2 so we have a peer - admin_set_node_url $admin_port 2 "https://node2.example.com:8011" - sleep 1 - - # Set node 2 status to "down" - log_info "Setting node 2 status to 'down' via Admin.SetNodeStatus..." - local response=$(admin_set_node_status $admin_port 2 "down") - log_info "SetNodeStatus response: $response" - - # Check if the response contains an error - if echo "$response" | grep -q '"error"'; then - log_error "SetNodeStatus returned error: $response" - return 1 - fi - - sleep 1 - - # Set node 2 status back to "up" - log_info "Setting node 2 status to 'up' via Admin.SetNodeStatus..." - response=$(admin_set_node_status $admin_port 2 "up") - log_info "SetNodeStatus response: $response" - - if echo "$response" | grep -q '"error"'; then - log_error "SetNodeStatus returned error: $response" - return 1 - fi - - # Test invalid status - log_info "Testing invalid status..." - response=$(admin_set_node_status $admin_port 2 "invalid") - if echo "$response" | grep -q '"error"'; then - log_info "Invalid status correctly rejected" - else - log_warn "Invalid status was not rejected (may be acceptable)" - fi - - log_info "Admin.SetNodeStatus test PASSED" - return 0 -} - -# ============================================================================= -# Test 18: Node down excluded from RegisterCvm response -# ============================================================================= -test_node_status_register_exclude() { - log_info "========== Test 18: Node Down Excluded from Registration ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" - - generate_config 1 - generate_config 2 - - start_node 1 - start_node 2 - - # Register peers so nodes can discover each other - setup_peers 1 2 - - local admin_port1=13016 - local admin_port2=13026 - local debug_port1=13015 - - # Wait for sync - sleep 5 - - # Verify debug service is available - if ! check_debug_service $debug_port1; then - log_error "Debug service not available on node 1" - return 1 - fi - - # Set node 2 status to "down" via node 1's admin API - log_info "Setting node 2 status to 'down'..." - admin_set_node_status $admin_port1 2 "down" - sleep 2 - - # Register a client on node 1 - log_info "Registering client on node 1 (node 2 is down)..." - local response=$(debug_register_cvm $debug_port1 "downtest12345678901234567890123456789012=" "downtest_app" "downtest_inst") - log_info "Register response: $response" - - # Verify registration succeeded - local client_ip=$(verify_register_response "$response") - if [[ -z "$client_ip" ]]; then - log_error "Registration failed" - return 1 - fi - log_info "Registered client with IP: $client_ip" - - # Check gateways list in response - should NOT include node 2 - local has_node2=$(echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - gateways = d.get('gateways', []) - for gw in gateways: - if gw.get('id') == 2: - sys.exit(0) - sys.exit(1) -except Exception: - sys.exit(1) -" && echo "yes" || echo "no") - - if [[ "$has_node2" == "yes" ]]; then - log_error "Node 2 (down) was included in registration response" - log_info "Response: $response" - return 1 - else - log_info "Node 2 (down) correctly excluded from registration response" - fi - - # Set node 2 status back to "up" - log_info "Setting node 2 status to 'up'..." - admin_set_node_status $admin_port1 2 "up" - sleep 2 - - # Register another client - log_info "Registering client on node 1 (node 2 is now up)..." - response=$(debug_register_cvm $debug_port1 "uptest123456789012345678901234567890123=" "uptest_app" "uptest_inst2") - - # Check gateways list - should now include node 2 - has_node2=$(echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - gateways = d.get('gateways', []) - for gw in gateways: - if gw.get('id') == 2: - sys.exit(0) - sys.exit(1) -except Exception: - sys.exit(1) -" && echo "yes" || echo "no") - - if [[ "$has_node2" == "no" ]]; then - log_error "Node 2 (up) was NOT included in registration response" - log_info "Response: $response" - return 1 - else - log_info "Node 2 (up) correctly included in registration response" - fi - - log_info "Node down excluded from registration test PASSED" - return 0 -} - -# ============================================================================= -# Test 19: Node down rejects RegisterCvm requests -# ============================================================================= -test_node_status_register_reject() { - log_info "========== Test 19: Node Down Rejects Registration ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" - - generate_config 1 - start_node 1 - - local admin_port=13016 - local debug_port=13015 - - # Verify debug service is available - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - - # Register a client when node is up (should succeed) - log_info "Registering client when node 1 is up..." - local response=$(debug_register_cvm $debug_port "upnode123456789012345678901234567890123=" "upnode_app" "upnode_inst") - local client_ip=$(verify_register_response "$response") - if [[ -z "$client_ip" ]]; then - log_error "Registration failed when node was up" - return 1 - fi - log_info "Registration succeeded when node was up (IP: $client_ip)" - - # Set node 1 status to "down" (marking itself as down) - log_info "Setting node 1 status to 'down'..." - admin_set_node_status $admin_port 1 "down" - sleep 2 - - # Try to register a client when node is down (should fail) - log_info "Attempting to register client when node 1 is down..." - response=$(debug_register_cvm $debug_port "downnode12345678901234567890123456789012=" "downnode_app" "downnode_inst") - log_info "Register response: $response" - - # Check if response contains error about node being down - if echo "$response" | grep -qi "error"; then - log_info "Registration correctly rejected when node is down" - if echo "$response" | grep -qi "marked as down"; then - log_info "Error message mentions 'marked as down' (correct)" - fi - else - log_error "Registration was NOT rejected when node is down" - log_info "Response: $response" - return 1 - fi - - # Set node 1 status back to "up" - log_info "Setting node 1 status to 'up'..." - admin_set_node_status $admin_port 1 "up" - sleep 2 - - # Register a client again (should succeed) - log_info "Registering client when node 1 is back up..." - response=$(debug_register_cvm $debug_port "backup123456789012345678901234567890123=" "backup_app" "backup_inst") - client_ip=$(verify_register_response "$response") - if [[ -z "$client_ip" ]]; then - log_error "Registration failed when node was back up" - return 1 - fi - log_info "Registration succeeded when node was back up (IP: $client_ip)" - - log_info "Node down rejects registration test PASSED" - return 0 -} - -# ============================================================================= -# Clean command - remove all generated files -# ============================================================================= -clean() { - log_info "Cleaning up generated files..." - - # Kill only test gateway processes (matching our specific config path) - pkill -9 -f "dstack-gateway -c ${SCRIPT_DIR}/${RUN_DIR}/node" >/dev/null 2>&1 || true - pkill -9 -f "dstack-gateway.*${SCRIPT_DIR}/${RUN_DIR}/node" >/dev/null 2>&1 || true - sleep 1 - - # Remove WireGuard interfaces (only our test interfaces need sudo) - sudo ip link delete wavekv-test1 2>/dev/null || true - sudo ip link delete wavekv-test2 2>/dev/null || true - sudo ip link delete wavekv-test3 2>/dev/null || true - - # Remove run directory (contains all generated files including certs) - rm -rf "$RUN_DIR" - - log_info "Cleanup complete" -} - -# ============================================================================= -# Ensure all certificates exist (CA + RPC + proxy) -# ============================================================================= -ensure_certs() { - # Create directories - mkdir -p "$CERTS_DIR" - mkdir -p "$RUN_DIR/certbot/live" - - # Generate CA certificate if not exists - if [[ ! -f "$CERTS_DIR/gateway-ca.key" ]] || [[ ! -f "$CERTS_DIR/gateway-ca.cert" ]]; then - log_info "Creating CA certificate..." - openssl genrsa -out "$CERTS_DIR/gateway-ca.key" 2048 2>/dev/null - openssl req -x509 -new -nodes \ - -key "$CERTS_DIR/gateway-ca.key" \ - -sha256 -days 365 \ - -out "$CERTS_DIR/gateway-ca.cert" \ - -subj "/CN=Test CA/O=WaveKV Test" \ - 2>/dev/null - fi - - # Generate RPC certificate signed by CA if not exists - if [[ ! -f "$CERTS_DIR/gateway-rpc.key" ]] || [[ ! -f "$CERTS_DIR/gateway-rpc.cert" ]]; then - log_info "Creating RPC certificate signed by CA..." - openssl genrsa -out "$CERTS_DIR/gateway-rpc.key" 2048 2>/dev/null - openssl req -new \ - -key "$CERTS_DIR/gateway-rpc.key" \ - -out "$CERTS_DIR/gateway-rpc.csr" \ - -subj "/CN=localhost" \ - 2>/dev/null - # Create certificate with SAN for localhost - cat >"$CERTS_DIR/ext.cnf" <<EXTEOF -authorityKeyIdentifier=keyid,issuer -basicConstraints=CA:FALSE -keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment -subjectAltName = @alt_names - -[alt_names] -DNS.1 = localhost -IP.1 = 127.0.0.1 -EXTEOF - openssl x509 -req \ - -in "$CERTS_DIR/gateway-rpc.csr" \ - -CA "$CERTS_DIR/gateway-ca.cert" \ - -CAkey "$CERTS_DIR/gateway-ca.key" \ - -CAcreateserial \ - -out "$CERTS_DIR/gateway-rpc.cert" \ - -days 365 \ - -sha256 \ - -extfile "$CERTS_DIR/ext.cnf" \ - 2>/dev/null - rm -f "$CERTS_DIR/gateway-rpc.csr" "$CERTS_DIR/ext.cnf" - fi - - # Generate proxy certificates (for TLS termination) - local proxy_cert_dir="$RUN_DIR/certbot/live" - if [[ ! -f "$proxy_cert_dir/cert.pem" ]] || [[ ! -f "$proxy_cert_dir/key.pem" ]]; then - log_info "Creating proxy certificates..." - openssl req -x509 -newkey rsa:2048 -nodes \ - -keyout "$proxy_cert_dir/key.pem" \ - -out "$proxy_cert_dir/cert.pem" \ - -days 365 \ - -subj "/CN=localhost" \ - 2>/dev/null - fi -} - -# ============================================================================= -# Main -# ============================================================================= -main() { - # Handle clean command - if [[ "${1:-}" == "clean" ]]; then - clean - exit 0 - fi - - # Handle cfg command - generate node configuration - if [[ "${1:-}" == "cfg" ]]; then - local node_id="${2:-}" - if [[ -z "$node_id" ]]; then - log_error "Usage: $0 cfg <node_id>" - log_info "Example: $0 cfg 1" - exit 1 - fi - - # Ensure certificates exist - ensure_certs - - # Generate config for the specified node - generate_config "$node_id" - log_info "Configuration generated: $RUN_DIR/node${node_id}.toml" - exit 0 - fi - - # Handle ls command - list all test cases - if [[ "${1:-}" == "ls" ]]; then - echo "Available test cases:" - echo "" - echo "Quick tests:" - echo " test_persistence - Single node persistence" - echo " test_status_endpoint - Status endpoint structure" - echo " test_prpc_register - prpc DebugRegisterCvm endpoint" - echo " test_prpc_info - prpc Info endpoint" - echo " test_wal_integrity - WAL file integrity" - echo "" - echo "Sync tests:" - echo " test_multi_node_sync - Multi-node sync" - echo " test_node_recovery - Node recovery after disconnect" - echo " test_cross_node_data_sync - Cross-node data sync verification" - echo "" - echo "Advanced tests:" - echo " test_client_registration_persistence - Client registration and persistence" - echo " test_stress_writes - Stress test - multiple writes" - echo " test_network_partition - Network partition simulation" - echo " test_three_node_cluster - Three-node cluster" - echo " test_three_node_bootnode - Three-node cluster with bootnode" - echo " test_node_id_reuse_rejected - Node ID reuse rejection" - echo " test_periodic_persistence - Periodic persistence" - echo "" - echo "Admin RPC tests:" - echo " test_admin_set_node_url - Admin.SetNodeUrl RPC" - echo " test_admin_set_node_status - Admin.SetNodeStatus RPC" - echo " test_node_status_register_exclude - Node down excluded from registration" - echo " test_node_status_register_reject - Node down rejects registration" - echo "" - echo "Usage:" - echo " $0 - Run all tests" - echo " $0 quick - Run quick tests only" - echo " $0 sync - Run sync tests only" - echo " $0 advanced - Run advanced tests only" - echo " $0 admin - Run admin RPC tests only" - echo " $0 case <name> - Run specific test case" - echo " $0 ls - List all test cases" - echo " $0 clean - Clean up generated files" - exit 0 - fi - - # Handle case command - run specific test case - if [[ "${1:-}" == "case" ]]; then - local test_case="${2:-}" - if [[ -z "$test_case" ]]; then - log_error "Usage: $0 case <testcase>" - log_info "Run '$0 ls' to see all available test cases" - exit 1 - fi - - # Check if gateway binary exists - if [[ ! -f "$GATEWAY_BIN" ]]; then - log_error "Gateway binary not found: $GATEWAY_BIN" - log_info "Please run: cargo build --release" - exit 1 - fi - - # Ensure certificates exist - ensure_certs - - # Check if test function exists - if ! declare -f "$test_case" >/dev/null; then - log_error "Test case not found: $test_case" - log_info "Use '$0 case' to see available test cases" - exit 1 - fi - - # Run the specific test - log_info "Running test case: $test_case" - CURRENT_TEST="$test_case" - if $test_case; then - log_info "Test PASSED: $test_case" - cleanup - exit 0 - else - log_error "Test FAILED: $test_case" - cleanup - exit 1 - fi - fi - - log_info "Starting WaveKV integration tests..." - - if [[ ! -f "$GATEWAY_BIN" ]]; then - log_error "Gateway binary not found: $GATEWAY_BIN" - log_info "Please run: cargo build --release" - exit 1 - fi - - # Ensure all certificates exist (RPC + proxy) - ensure_certs - - local failed=0 - local passed=0 - local failed_tests=() - - run_test() { - local test_name=$1 - CURRENT_TEST="$test_name" - if $test_name; then - ((passed++)) - else - ((failed++)) - failed_tests+=("$test_name") - fi - cleanup - } - - # Run selected test or all tests - local test_filter="${1:-all}" - - if [[ "$test_filter" == "all" ]] || [[ "$test_filter" == "quick" ]]; then - run_test test_persistence - run_test test_status_endpoint - run_test test_prpc_register - run_test test_prpc_info - run_test test_wal_integrity - fi - - if [[ "$test_filter" == "all" ]] || [[ "$test_filter" == "sync" ]]; then - run_test test_multi_node_sync - run_test test_node_recovery - run_test test_cross_node_data_sync - fi - - if [[ "$test_filter" == "all" ]] || [[ "$test_filter" == "advanced" ]]; then - run_test test_client_registration_persistence - run_test test_stress_writes - run_test test_network_partition - run_test test_three_node_cluster - run_test test_three_node_bootnode - run_test test_node_id_reuse_rejected - run_test test_periodic_persistence - fi - - if [[ "$test_filter" == "all" ]] || [[ "$test_filter" == "admin" ]]; then - run_test test_admin_set_node_url - run_test test_admin_set_node_status - run_test test_node_status_register_exclude - run_test test_node_status_register_reject - fi - - echo "" - log_info "==========================================" - log_info "Tests passed: $passed" - if [[ $failed -gt 0 ]]; then - log_error "Tests failed: $failed" - echo "" - log_error "Failed test cases:" - for test_name in "${failed_tests[@]}"; do - log_error " - $test_name" - done - echo "" - log_info "To rerun a failed test:" - log_info " $0 case <test_name>" - log_info "Example:" - if [[ ${#failed_tests[@]} -gt 0 ]]; then - log_info " $0 case ${failed_tests[0]}" - fi - fi - log_info "==========================================" - - return $failed -} - -# Run if executed directly -if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - main "$@" -fi diff --git a/dstack/guest-agent-simulator/Cargo.toml b/dstack/guest-agent-simulator/Cargo.toml index f476e01c2..63855dd60 100644 --- a/dstack/guest-agent-simulator/Cargo.toml +++ b/dstack/guest-agent-simulator/Cargo.toml @@ -22,6 +22,11 @@ tracing.workspace = true tracing-subscriber.workspace = true rocket.workspace = true ra-rpc = { workspace = true, features = ["rocket"] } -ra-tls = { workspace = true, features = ["quote"] } +ra-tls = { workspace = true, features = ["quote", "simulator"] } dstack-guest-agent = { path = "../guest-agent" } dstack-guest-agent-rpc.workspace = true +dstack-types.workspace = true +cc-eventlog.workspace = true +dcap-qvl.workspace = true +hex.workspace = true +mock-attestation = { path = "../crates/mock-attestation" } diff --git a/dstack/guest-agent-simulator/src/main.rs b/dstack/guest-agent-simulator/src/main.rs index c2bf89bf1..8807d167f 100644 --- a/dstack/guest-agent-simulator/src/main.rs +++ b/dstack/guest-agent-simulator/src/main.rs @@ -13,7 +13,8 @@ use dstack_guest_agent::{ config::{self, Config}, run_server, AppState, }; -use dstack_guest_agent_rpc::{AttestResponse, GetQuoteResponse}; +use dstack_guest_agent_rpc::v0::GetQuoteResponse; +use mock_attestation::tdx::TdxGenerator; use ra_tls::attestation::VersionedAttestation; use serde::Deserialize; use tracing::warn; @@ -37,6 +38,8 @@ struct SimulatorSettings { attestation_file: String, #[serde(default = "default_patch_report_data")] patch_report_data: bool, + #[serde(default)] + mock_attestation_seed: Option<String>, } #[derive(Debug, Clone, Deserialize)] @@ -49,14 +52,25 @@ struct SimulatorCoreConfig { struct SimulatorPlatform { attestation: VersionedAttestation, patch_report_data: bool, + generator: Option<TdxGenerator>, } impl SimulatorPlatform { - fn new(attestation: VersionedAttestation, patch_report_data: bool) -> Self { - Self { + fn new( + attestation: VersionedAttestation, + patch_report_data: bool, + mock_attestation_seed: Option<&str>, + ) -> Result<Self> { + let generator = mock_attestation_seed + .map(mock_attestation::parse_seed) + .transpose()? + .map(TdxGenerator::from_seed) + .transpose()?; + Ok(Self { attestation, patch_report_data, - } + generator, + }) } } @@ -74,6 +88,7 @@ impl PlatformBackend for SimulatorPlatform { &self.attestation, pubkey, self.patch_report_data, + self.generator.as_ref(), ) } @@ -83,11 +98,17 @@ impl PlatformBackend for SimulatorPlatform { report_data, vm_config, self.patch_report_data, + self.generator.as_ref(), ) } - fn attest_response(&self, report_data: [u8; 64]) -> Result<AttestResponse> { - simulator::simulated_attest_response(&self.attestation, report_data, self.patch_report_data) + fn attest_cvm(&self, report_data: [u8; 64]) -> Result<VersionedAttestation> { + simulator::simulated_attest_response( + &self.attestation, + report_data, + self.patch_report_data, + self.generator.as_ref(), + ) } } @@ -107,12 +128,17 @@ async fn main() -> Result<()> { warn!( attestation_file = %sim_config.simulator.attestation_file, patch_report_data = sim_config.simulator.patch_report_data, + signed_quotes = sim_config.simulator.mock_attestation_seed.is_some(), "starting dstack guest-agent simulator" ); if sim_config.simulator.patch_report_data { - warn!("simulator will rewrite report_data to match requests; quote verification may fail against the original fixture signature"); + warn!( + "simulator will rewrite report_data to match requests; quote verification may fail against the original fixture signature" + ); } else { - warn!("simulator will preserve fixture report_data; cert/key binding and requested report_data may not match"); + warn!( + "simulator will preserve fixture report_data; cert/key binding and requested report_data may not match" + ); } let attestation = simulator::load_versioned_attestation(&sim_config.simulator.attestation_file)?; @@ -121,7 +147,8 @@ async fn main() -> Result<()> { Arc::new(SimulatorPlatform::new( attestation, sim_config.simulator.patch_report_data, - )), + sim_config.simulator.mock_attestation_seed.as_deref(), + )?), ) .await .context("Failed to create simulator app state")?; @@ -131,6 +158,7 @@ async fn main() -> Result<()> { #[cfg(test)] mod tests { use super::*; + use ra_tls::attestation::TdxAttestationExt; fn load_fixture_platform() -> SimulatorPlatform { let fixture = simulator::load_versioned_attestation( @@ -138,7 +166,7 @@ mod tests { .join("../guest-agent/fixtures/attestation.bin"), ) .expect("fixture attestation should load"); - SimulatorPlatform::new(fixture, true) + SimulatorPlatform::new(fixture, true, None).unwrap() } #[test] @@ -152,16 +180,132 @@ mod tests { } #[test] - fn simulator_attest_response_uses_supplied_report_data() { + fn simulator_attest_response_preserves_legacy_wire_format() { let platform = load_fixture_platform(); let report_data = [0x5a; 64]; - let response = platform.attest_response(report_data).unwrap(); - let patched = VersionedAttestation::from_bytes(&response.attestation) + let encoded = platform + .attest_cvm(report_data) + .unwrap() + .to_bytes() + .unwrap(); + assert_eq!(encoded.first(), Some(&0x00)); + let patched = VersionedAttestation::from_bytes(&encoded) .unwrap() .into_v1(); assert_eq!(patched.report_data().unwrap(), report_data); } + #[test] + fn seeded_simulator_resigns_certificate_attestation() { + let fixture = simulator::load_versioned_attestation( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../guest-agent/fixtures/attestation.bin"), + ) + .unwrap(); + let seed = [0x5a; 32]; + let platform = SimulatorPlatform::new(fixture, true, Some(&hex::encode(seed))).unwrap(); + let attestation = platform + .certificate_attestation(b"test-public-key") + .unwrap() + .into_v1(); + let quote = attestation.tdx_quote_bytes().unwrap(); + let generator = TdxGenerator::from_seed(seed).unwrap(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + dcap_qvl::verify::QuoteVerifier::new(generator.root_ca_der()) + .verify("e, &generator.sample_collateral().unwrap(), now) + .unwrap(); + assert_eq!( + attestation.report_data().unwrap(), + ra_tls::attestation::QuoteContentType::RaTlsCert.to_report_data(b"test-public-key") + ); + } + + #[test] + fn simulator_rejects_get_quote_on_non_tdx() { + use ra_tls::attestation::PlatformEvidence; + + let fixture = simulator::load_versioned_attestation( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../guest-agent/fixtures/attestation.bin"), + ) + .expect("fixture attestation should load"); + let mut attestation = fixture.into_v1(); + attestation.platform = PlatformEvidence::SevSnp { + report: vec![0u8; 1184], + cert_chain: Vec::new(), + mr_config: String::new(), + }; + let non_tdx = VersionedAttestation::V1 { attestation }; + let report_data = [0x5a; 64]; + + // GetQuote is Intel TDX only. + let err = simulator::simulated_quote_response(&non_tdx, report_data, "", true, None) + .expect_err("GetQuote must fail on a non-TDX platform"); + assert!( + err.to_string().contains("Intel TDX only"), + "unexpected error: {err}" + ); + + // Attest remains the supported path on the same platform. + simulator::simulated_attest_response(&non_tdx, report_data, true, None) + .expect("Attest must still work on a non-TDX platform"); + } + + #[test] + fn simulator_serves_get_quote_on_gcp_tdx() { + use dstack_types::Platform; + use ra_tls::attestation::{PlatformEvidence, TpmQuote}; + + let fixture = simulator::load_versioned_attestation( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../guest-agent/fixtures/attestation.bin"), + ) + .expect("fixture attestation should load"); + let mut attestation = fixture.into_v1(); + let (quote, event_log) = match attestation.platform { + PlatformEvidence::Tdx { quote, event_log } => (quote, event_log), + other => panic!("fixture should carry bare TDX evidence, got {other:?}"), + }; + attestation.platform = PlatformEvidence::GcpTdx { + quote, + event_log, + tpm_quote: TpmQuote { + message: Vec::new(), + signature: Vec::new(), + pcr_values: Vec::new(), + ak_cert: Vec::new(), + platform: Platform::Gcp, + event_log: Vec::new(), + }, + }; + let gcp_tdx = VersionedAttestation::V1 { attestation }; + let report_data = [0x5a; 64]; + + // The gate is "does this platform have a TDX quote", not "is this bare + // TDX", so GCP Confidential VMs are served, with the report data + // patched into the quote the same way bare TDX gets it. + let response = simulator::simulated_quote_response(&gcp_tdx, report_data, "", true, None) + .expect("GetQuote must answer on GCP TDX"); + assert_eq!( + &response.quote[ra_tls::attestation::TDX_QUOTE_REPORT_DATA_RANGE], + &report_data + ); + assert_eq!(response.report_data, report_data); + + // What the response cannot carry is the vTPM quote GCP's verification + // also binds -- it has no field for one. That is why the docs point + // relying parties on GCP at Attest. + let attested = simulator::simulated_attest_response(&gcp_tdx, report_data, true, None) + .expect("Attest must work on GCP TDX too"); + let round_tripped = VersionedAttestation::from_bytes(&attested.to_bytes().unwrap()) + .unwrap() + .into_v1(); + assert!(round_tripped.platform.tpm_quote().is_some()); + } + #[test] fn simulator_can_preserve_fixture_report_data() { let fixture = simulator::load_versioned_attestation( @@ -170,10 +314,14 @@ mod tests { ) .expect("fixture attestation should load"); let original = fixture.clone().into_v1().report_data().unwrap(); - let platform = SimulatorPlatform::new(fixture, false); + let platform = SimulatorPlatform::new(fixture, false, None).unwrap(); let report_data = [0x5a; 64]; - let response = platform.attest_response(report_data).unwrap(); - let patched = VersionedAttestation::from_bytes(&response.attestation) + let encoded = platform + .attest_cvm(report_data) + .unwrap() + .to_bytes() + .unwrap(); + let patched = VersionedAttestation::from_bytes(&encoded) .unwrap() .into_v1(); assert_eq!(patched.report_data().unwrap(), original); diff --git a/dstack/guest-agent-simulator/src/simulator.rs b/dstack/guest-agent-simulator/src/simulator.rs index ed148429b..35652b387 100644 --- a/dstack/guest-agent-simulator/src/simulator.rs +++ b/dstack/guest-agent-simulator/src/simulator.rs @@ -5,9 +5,11 @@ use std::path::Path; use anyhow::{anyhow, Context, Result}; -use dstack_guest_agent_rpc::{AttestResponse, GetQuoteResponse}; +use dcap_qvl::quote::Quote; +use dstack_guest_agent_rpc::v0::GetQuoteResponse; +use mock_attestation::tdx::TdxGenerator; use ra_tls::attestation::{ - AttestationV1, QuoteContentType, TdxAttestationExt, VersionedAttestation, + AttestationV1, PlatformEvidence, QuoteContentType, TdxAttestationExt, VersionedAttestation, }; use std::fs; use tracing::warn; @@ -29,34 +31,45 @@ pub fn simulated_quote_response( report_data: [u8; 64], vm_config: &str, patch_report_data: bool, + generator: Option<&TdxGenerator>, ) -> Result<GetQuoteResponse> { - let attestation = maybe_patch_report_data(attestation, report_data, patch_report_data, "quote"); + let attestation = prepare_attestation( + attestation, + report_data, + patch_report_data, + generator, + "quote", + )?; let Some(quote) = attestation.tdx_quote_bytes() else { - return Err(anyhow!("Quote not found")); + return Err(anyhow!( + "GetQuote is Intel TDX only, use Attest on this platform" + )); }; - let versioned = VersionedAttestation::V1 { - attestation: attestation.clone(), - } - .to_bytes()?; Ok(GetQuoteResponse { quote, event_log: attestation.tdx_event_log_string().unwrap_or_default(), report_data: report_data.to_vec(), vm_config: vm_config.to_string(), - attestation: versioned, }) } pub fn simulated_attest_response( - attestation: &VersionedAttestation, + source: &VersionedAttestation, report_data: [u8; 64], patch_report_data: bool, -) -> Result<AttestResponse> { - let attestation = - maybe_patch_report_data(attestation, report_data, patch_report_data, "attest"); - Ok(AttestResponse { - attestation: VersionedAttestation::V1 { attestation }.to_bytes()?, + generator: Option<&TdxGenerator>, +) -> Result<VersionedAttestation> { + let preserve_legacy = matches!(source, VersionedAttestation::V0 { .. }); + let mut attestation = + prepare_attestation(source, report_data, patch_report_data, generator, "attest")?; + if let Some(event_log) = attestation.platform.tdx_event_log_mut() { + cc_eventlog::tdx::fill_v2_preimages(event_log); + } + Ok(if preserve_legacy { + attestation.try_into_legacy()?.into_versioned() + } else { + VersionedAttestation::V1 { attestation } }) } @@ -65,20 +78,68 @@ pub fn simulated_info_attestation(attestation: &VersionedAttestation) -> Version } pub fn simulated_certificate_attestation( - attestation: &VersionedAttestation, + source: &VersionedAttestation, pubkey: &[u8], patch_report_data: bool, + generator: Option<&TdxGenerator>, ) -> Result<VersionedAttestation> { + let preserve_legacy = matches!(source, VersionedAttestation::V0 { .. }); let report_data = QuoteContentType::RaTlsCert.to_report_data(pubkey); - let attestation = maybe_patch_report_data( - attestation, + let attestation = prepare_attestation( + source, report_data, patch_report_data, + generator, "certificate_attestation", - ); + )?; + if preserve_legacy { + return Ok(attestation.try_into_legacy()?.into_versioned()); + } Ok(VersionedAttestation::V1 { attestation }) } +fn prepare_attestation( + attestation: &VersionedAttestation, + report_data: [u8; 64], + patch_report_data: bool, + generator: Option<&TdxGenerator>, + context: &str, +) -> Result<AttestationV1> { + let Some(generator) = generator else { + return Ok(maybe_patch_report_data( + attestation, + report_data, + patch_report_data, + context, + )); + }; + // Stack half only: the fixture quote read below is replaced outright by + // the generated one, so patching its report data would be undone. + let mut attestation = attestation + .clone() + .into_v1() + .with_stack_report_data(report_data); + let quote = attestation + .platform + .tdx_quote() + .context("TDX quote is unavailable in simulator fixture")?; + let quote = Quote::parse(quote).context("invalid simulator fixture TDX quote")?; + let report = quote + .report + .as_td10() + .context("simulator fixture does not contain a TDX 1.0 report")?; + let evidence = generator.attest_with_measurements( + report_data, + report.mr_td, + [report.rt_mr0, report.rt_mr1, report.rt_mr2, report.rt_mr3], + )?; + match &mut attestation.platform { + PlatformEvidence::Tdx { quote, .. } => *quote = evidence.quote, + _ => return Err(anyhow!("seeded simulator requires dstack TDX evidence")), + } + Ok(attestation) +} + fn maybe_patch_report_data( attestation: &VersionedAttestation, report_data: [u8; 64], diff --git a/dstack/guest-agent/Cargo.toml b/dstack/guest-agent/Cargo.toml index 8f2235f2e..9b028a6cc 100644 --- a/dstack/guest-agent/Cargo.toml +++ b/dstack/guest-agent/Cargo.toml @@ -10,6 +10,7 @@ edition.workspace = true license.workspace = true [dependencies] +nvattest.workspace = true rocket.workspace = true tracing.workspace = true tracing-subscriber.workspace = true @@ -19,14 +20,15 @@ fs-err.workspace = true rcgen.workspace = true sha2.workspace = true clap.workspace = true -tokio.workspace = true +tokio = { workspace = true, features = ["process"] } hex.workspace = true serde_json.workspace = true bollard.workspace = true +futures.workspace = true chrono.workspace = true base64.workspace = true rinja.workspace = true -git-version.workspace = true +dstack-build-info.workspace = true ra-rpc = { workspace = true, features = ["client", "rocket"] } dstack-guest-agent-rpc.workspace = true ra-tls = { workspace = true, features = ["quote"] } @@ -55,3 +57,12 @@ rand.workspace = true or-panic.workspace = true cc-eventlog.workspace = true listenfd.workspace = true +libc.workspace = true +cached-cell.workspace = true +lspci.workspace = true + +[dev-dependencies] +# The test-only mock platform builds attestations from a fixture, which needs +# the synthetic-attestation helpers. Declared here so the production build of +# this crate does not carry them. +dstack-attest = { workspace = true, features = ["simulator"] } diff --git a/dstack/guest-agent/rpc/Cargo.toml b/dstack/guest-agent/rpc/Cargo.toml index a7e573c64..7f3d1f877 100644 --- a/dstack/guest-agent/rpc/Cargo.toml +++ b/dstack/guest-agent/rpc/Cargo.toml @@ -17,5 +17,10 @@ serde_json.workspace = true anyhow.workspace = true scale.workspace = true +[dev-dependencies] +prost-types.workspace = true +sha2.workspace = true +hex.workspace = true + [build-dependencies] prpc-build.workspace = true diff --git a/dstack/guest-agent/rpc/build.rs b/dstack/guest-agent/rpc/build.rs index bc584fdbe..fe19530a5 100644 --- a/dstack/guest-agent/rpc/build.rs +++ b/dstack/guest-agent/rpc/build.rs @@ -11,10 +11,6 @@ fn main() { .build_scale_ext(false) .disable_package_emission() .enable_serde_extension() - .field_attribute( - ".dstack_guest.GetQuoteResponse.attestation", - "#[serde(skip_serializing_if = \"::prost::alloc::vec::Vec::is_empty\")]", - ) .disable_service_name_emission() .compile_dir("./proto") .expect("failed to compile proto files"); diff --git a/dstack/guest-agent/rpc/proto/agent_rpc.proto b/dstack/guest-agent/rpc/proto/agent_rpc.proto index 892cca70b..6f0a44c95 100644 --- a/dstack/guest-agent/rpc/proto/agent_rpc.proto +++ b/dstack/guest-agent/rpc/proto/agent_rpc.proto @@ -34,7 +34,27 @@ service Tappd { rpc Version(google.protobuf.Empty) returns (WorkerVersion) {} } -// The service for the dstack guest agent +// The service for the dstack guest agent. +// +// This surface is CLOSED. It is exactly the v0.5.11 surface and stays that way: +// no additions, no renumbering, no removals, and no semantic changes to +// existing methods. It exists so a v0.5.x client keeps working against a 0.6 +// agent unchanged. +// +// It is the `v0` surface, served at `/v0`. Its historical unversioned path `/` +// stays mounted as an alias onto this same handler for pre-0.6 clients. +// +// Every new capability goes to `dstack.guest.v1` (agent_rpc_v1.proto), served +// at `/v1` on the same socket. Do not add a method here, and do not add a +// field to a message here, even a wire-compatible one -- "frozen except for +// additions" is how this surface acquired `GpuInfo`, `AttestGpu` and +// `AttestArgs` between v0.5.11 and 0.6.0, none of which ever shipped and all +// of which have been moved to v1. +// +// Two behaviour-only changes are sanctioned and do not alter the wire shape: +// `GetQuote` now fails on a platform without Intel TDX instead of returning an +// empty quote, and `GetTlsKey` rejects a not_before that is not earlier than +// not_after. `EmitEvent` always fails; see its doc comment. service DstackGuest { // Derives a cryptographic key from the specified key path. // Returns the derived key along with its TLS certificate chain. @@ -44,19 +64,45 @@ service DstackGuest { rpc GetKey(GetKeyArgs) returns (GetKeyResponse) {} // Generates a TDX quote with given report data. + // + // Answers wherever the platform has Intel TDX; anywhere else it fails, and + // Attest is the replacement. On GCP Confidential VMs it answers with the TDX + // quote alone, leaving out the vTPM quote GCP's verification also binds -- + // use Attest there to get evidence a verifier can check in full. rpc GetQuote(RawQuoteArgs) returns (GetQuoteResponse) {} // Generates a versioned attestation with the given report data. // Returns a dstack-defined attestation format that supports different attestation modes across platforms. rpc Attest(RawQuoteArgs) returns (AttestResponse) {} + // Removed in v0.6.0: always fails. Runtime RTMR3 events are system-owned now, + // so an app can no longer extend them. + // + // The method is kept only so a pre-0.6 client gets a self-explanatory error. + // Deleting it would answer HTTP 404 `Service not found: EmitEvent`, which + // says nothing about why the events stopped being recorded; the kept stub + // fails with HTTP 400 and a message naming the removal and what to do + // instead. The self-explanatory error is the whole point of keeping it. + rpc EmitEvent(EmitEventArgs) returns (google.protobuf.Empty) {} + // Get app info rpc Info(google.protobuf.Empty) returns (AppInfo) {} // Sign a payload rpc Sign(SignRequest) returns (SignResponse) {} - // Verify a signature + // Verify a signature. Deprecated, legacy-only. + // + // Verification needs no key material and no attestation, and the agent's + // answer arrives over the socket unattested, so a caller gains nothing over + // checking the signature itself. It is retained because it is part of this + // frozen surface, and the SDKs' v0 clients still expose it. + // + // v1 has no counterpart. A relying party verifies locally, against the rules + // in `docs/guest-api-v1.md`. + // + // Retained here only so 0.5.x clients keep working. It will not appear in the + // v1 API; do not call it from new code. rpc Verify(VerifyRequest) returns (VerifyResponse) {} // Get the guest agent version @@ -139,8 +185,11 @@ message GetKeyResponse { } // The request to get a TDX quote -// The report data is prefixed with `app-data:` before hashing unless the algorithm is `raw`. -// Final report data is hash(`app-data:` + report_data) if the algorithm is not `raw`. +// Unless the algorithm is `raw`, the final report data is +// hash(prefix + ":" + report_data), left-aligned in 64 bytes and zero-padded. +// The agent supplies the colon, so `prefix` is the tag alone and its default is +// `app-data` -- which is what the response echoes back. Passing `app-data:` as +// the prefix produces hash(`app-data::` + report_data), not the default. message TdxQuoteArgs { // Report data bytes report_data = 1; @@ -167,6 +216,15 @@ message TdxQuoteArgs { message RawQuoteArgs { // 64 bytes of report data bytes report_data = 1; + + // Between v0.5.11 and 0.6.0, `Attest` briefly took its own `AttestArgs` + // carrying `include_ccel` (2), `include_preimages` (3) and + // `include_boottime_gpu_evidence` (4). None of that shipped in a release, + // and `Attest` takes this message again, but an interim `next` build in + // somebody's dev environment may still send those numbers. Reserved so a + // future field cannot silently absorb one. + reserved 2, 3, 4; + reserved "include_ccel", "include_preimages", "include_boottime_gpu_evidence"; } message TdxQuoteResponse { @@ -185,21 +243,36 @@ message TdxQuoteResponse { message AttestResponse { // The attestation bytes attestation = 1; + + // Field 2 briefly carried `boottime_gpu_evidence` on `next` and never + // shipped. `dstack.guest.v1`'s own AttestResponse serves those bytes, under + // the same name, on `/v1/Attest`. + reserved 2; + reserved "boottime_gpu_evidence"; } message GetQuoteResponse { - // TDX quote (empty on non-TDX platforms such as AMD SEV-SNP) + reserved 5; + reserved "attestation"; + + // TDX quote bytes quote = 1; - // Event log (empty on non-TDX platforms) + // Event log. V2 runtime events always include their hex-encoded digest + // preimage. Clients should verify sha384(hex_decode(preimage)) == digest. string event_log = 2; // Report data bytes report_data = 3; // Hw config string vm_config = 4; - // Platform-adaptive versioned attestation (SCALE/msgpack encoded). Populated - // on non-TDX TEE platforms (AMD SEV-SNP, ...). TDX uses `quote` + `event_log` - // above to keep this response compact. - bytes attestation = 5; +} + +// The request of the removed EmitEvent RPC. Kept so a pre-0.6 client's request +// still decodes and reaches the handler that explains the removal. +message EmitEventArgs { + // The event name + string event = 1; + // The event data + bytes payload = 2; } // The request to derive a key @@ -240,17 +313,47 @@ message WorkerVersion { string rev = 2; } +// The external guest agent service. +// +// CLOSED, like `DstackGuest`: exactly the v0.5.11 surface, served at +// `/prpc/v0` on the external listener, with `/prpc` kept as an alias onto the +// same handler for pre-0.6 clients. New capability goes to `dstack.guest.v1`'s +// `dstack.guest.v1`'s `Worker`, served at `/prpc/v1`. service Worker { // Get app info rpc Info(google.protobuf.Empty) returns (AppInfo) {} // Get the guest agent version rpc Version(google.protobuf.Empty) returns (WorkerVersion) {} - // Get attestation + // Get attestation. + // + // Legacy and frozen. Returns a `GetQuoteResponse`, so it answers on Intel TDX + // and fails everywhere else. + // + // The v1 surface has no counterpart, deliberately: this attests the key v0's + // KDF derives at path `vms`, which no v1 `GetKey(domain, algorithm)` can + // return. A v1 application attests its own key instead -- derive it at + // `/v1/GetKey`, commit the public key into `report_data`, call `/v1/Attest`. rpc GetAttestationForAppKey(GetAttestationForAppKeyRequest) returns (GetQuoteResponse) {} } message SignRequest { + // Signing mode. One of `ed25519`, `secp256k1`, or `secp256k1_prehashed`. + // `k256` is an alias for `secp256k1`. There is no default: an empty string + // is rejected, unlike `GetKeyArgs.algorithm` which defaults to `secp256k1`. + // + // The key is always the one derived at path `vms` with purpose `signing`; + // `secp256k1_prehashed` derives under `secp256k1`, since prehashing is a + // signing mode and not a key type. string algorithm = 1; + // The payload. Raw bytes for `ed25519` and `secp256k1`, which the agent + // hashes itself -- RFC 8032 internally, SHA-256 respectively. + // + // For `secp256k1_prehashed` this is the digest, exactly 32 bytes, signed + // as-is: the agent does not hash it again and records nothing about which + // hash produced it, so the caller owns that choice and the domain + // separation that goes with it. + // + // Specified in docs/guest-api-v0.md. bytes data = 2; } diff --git a/dstack/guest-agent/rpc/proto/agent_rpc_v1.proto b/dstack/guest-agent/rpc/proto/agent_rpc_v1.proto new file mode 100644 index 000000000..da50b62d1 --- /dev/null +++ b/dstack/guest-agent/rpc/proto/agent_rpc_v1.proto @@ -0,0 +1,466 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package dstack.guest.v1; + +// The versioned dstack guest agent API. +// +// Two services, one per trust surface, mirroring the two unversioned ones: +// +// `DstackGuest` internal socket (`/var/run/dstack.sock`), at `/v1` +// `Worker` external listener, at `/prpc/v1` +// +// The names carry no version suffix: the package `dstack.guest.v1` already +// says which surface this is, and `dstack.guest.v1.DstackGuestV1` would say it +// twice. +// +// The frozen v0.5.11 services are named `v0` and served at `/v0` and +// `/prpc/v0`; their historical unversioned paths (`/` and `/prpc`) stay mounted +// as aliases onto the same handlers, so a pre-0.6 client keeps working. The +// version a caller gets is decided by the URL path alone, never by a header. +// +// The two are not the same surface with different mounts. The internal socket +// is reachable only by the application itself, so `DstackGuest` hands out key +// material; the external listener is reachable by anyone who can route to the +// CVM, so `Worker` never does. +// +// Conventions, applied without exception: +// - every request message is `<Method>Request`, every response +// `<Method>Response`, including for methods that take no arguments -- an +// empty message can gain a field, `google.protobuf.Empty` cannot; +// - a field's encoding is stated in its doc comment, not in its name. The +// fields that carry JSON documents say so and name who owns the schema; +// - a key is named by `(domain, algorithm)` and nothing else. There is no +// `purpose` field anywhere in v1. Encode roles in the domain string. +// +// v1 keys are NOT the v0 keys. The v1 KDF derives under its own HKDF salt and +// binds the algorithm and a versioned context tag alongside the domain, so the +// same name yields different key material here than on the frozen surface, and +// secp256k1 and ed25519 no longer share one 32-byte secret. See `docs/guest-api-v1.md` for the +// byte-level construction. +// +// v1 serves only what genuinely needs the TEE: deriving keys from the app root +// key, and attesting. It is not an HSM and does not pretend to be one. +// +// So there is no `Sign` and no `Verify`. Any caller that can reach this socket +// can ask `GetKey` for the private key itself, so a server-side `Sign` grants +// no capability the caller does not already have -- it is pure computation +// behind an IPC round trip, and one more entry point to audit. Verifying is +// the same argument without even the key: the agent's answer arrives +// unattested, so a relying party gains nothing over checking it itself. +// Applications sign locally with a standard library, using the key `GetKey` +// returns; `docs/guest-api-v1.md` specifies chain verification normatively. +// +// There is no `EmitEvent` either: runtime RTMR3 events are system-owned as of +// 0.6.0 and cannot be extended by an app. +// +// The unversioned surface keeps `Sign` and `Verify` for v0.5.x clients. They +// are frozen there and are not carried forward. +service DstackGuest { + // Issue a certificate for this application. + // + // The agent builds a CSR, signs it with the certificate's own key, and + // relays it to the KMS `SignCert` RPC (or to the local CA when the app runs + // without a KMS), then returns the chain the signer produced. Certificate + // issuance is the operation; v0 called this `GetTlsKey`, which named the + // by-product instead of the request. + // + // The private key is freshly generated for every call and is NOT derived + // from the app identity: none of the request fields feed it, and calling + // twice with the same arguments yields two unrelated keys. `GetKey` is the + // method that derives a stable, attestable key. + // + // This first cut serves only the integrated one-step mode, where the agent + // holds the key. A mode that signs a caller-supplied CSR or public key -- + // so the private key never leaves the caller -- is a plausible extension and + // would arrive as added fields or a sibling method, not as a change to what + // these fields mean. + rpc IssueCert(IssueCertRequest) returns (IssueCertResponse) {} + + // Derive an application key from `(domain, algorithm)` and return the private + // key with its signature chain. + rpc GetKey(GetKeyRequest) returns (GetKeyResponse) {} + + // Produce a versioned attestation over the given report data. + // + // The sole CVM attestation entry point in v1. The dstack-defined attestation + // format covers every supported platform and already carries the TDX quote + // and the event log, so v0's TDX-only `GetQuote` has nothing left to add: + // it answers on Intel TDX and nowhere else, and on GCP Confidential VMs it + // returns the TDX quote without the vTPM quote GCP's verification also + // binds. `docs/guest-api-v1.md` says how to extract a raw quote and event + // log from the attestation. `GetQuote` stays on the unversioned surface for + // v0.5.x clients. + rpc Attest(AttestRequest) returns (AttestResponse) {} + + // Collect GPU attestation evidence now, against a nonce the caller chooses. + // + // This answers "is the device I can talk to right now a genuine, CC-enabled + // NVIDIA GPU that signs my challenge", which `Attest`'s + // `boottime_gpu_evidence` cannot: that is the record nvattest wrote at boot, + // against its own nonce. Use this after anything that may have reinitialised + // the GPU -- a driver reload leaves a device that responds to NVML but can no + // longer attest -- and before submitting work you care about. + // + // Returns vendor-native evidence, not a local verdict, so a relying party + // can appraise it with its own verifier. Evidence still does not bind the + // GPU to this TD; see `AttestGpuResponse.bundles`. + // + // An image that ships no GPU attestation answers HTTP 501, not the 400 every + // other v1 failure uses. The request is well-formed there and no other + // request would succeed either, so a caller can tell "fall back" from "fix + // your arguments" on the status alone. A malformed nonce stays a 400. + rpc AttestGpu(AttestGpuRequest) returns (AttestGpuResponse) {} + + // Return this application's identity and measurements. + rpc Info(InfoRequest) returns (InfoResponse) {} + + // Return the guest agent version. + rpc Version(VersionRequest) returns (VersionResponse) {} +} + +// --------------------------------------------------------------------------- +// Certificate issuance +// --------------------------------------------------------------------------- + +message IssueCertRequest { + // Subject of the certificate to request. + string subject = 1; + // DNS alternative names for the certificate. + repeated string alt_names = 2; + // Include the attestation in the certificate (RA-TLS). The embedded + // attestation is always the MessagePack V1 schema, so the signing KMS and + // any verifying peer must be dstack 0.5.9 or later. + bool usage_ra_tls = 3; + // Key usage server auth. + bool usage_server_auth = 4; + // Key usage client auth. + bool usage_client_auth = 5; + // Certificate validity start, seconds since the UNIX epoch. Rejected when + // it is not earlier than `not_after`. + optional uint64 not_before = 6; + // Certificate validity end, seconds since the UNIX epoch. + optional uint64 not_after = 7; + // Include app info in the certificate. + bool with_app_info = 8; +} + +message IssueCertResponse { + // The private key the agent generated for this certificate, PEM-encoded. + // Fresh per call; see `IssueCert`. + string key = 1; + // The certificate chain, leaf first, each entry PEM-encoded, exactly as the + // signer returned it. + repeated string certificate_chain = 2; +} + +// --------------------------------------------------------------------------- +// Application keys +// --------------------------------------------------------------------------- + +message GetKeyRequest { + // Caller-chosen domain-separation string identifying the key. Not a DNS + // name -- the certificate fields on `IssueCertRequest` are the ones that + // take those. + // + // Any byte string a proto3 `string` can carry, including one with `:`, `/` + // or NUL in it: the KDF length-prefixes it, so no domain can be confused + // with another. + // + // Derivation is flat. Two domains yield unrelated keys, and `a/b` is not a + // child of `a` in any sense -- there is no BIP-32-style hierarchy and no + // parent key from which a sub-domain's key can be computed. + // + // This replaces v0's `path` plus `purpose`. In v0 only `path` reached the + // KDF and `purpose` was merely echoed into the chain claim; here the one + // field feeds both, alongside `algorithm`. + string domain = 1; + // Key type. Exactly `secp256k1` or `ed25519`. + // + // There is no default and no alias: an empty or unrecognised value is an + // error. v0 defaulted an empty string to secp256k1 and accepted `k256`, + // which meant a typo silently produced a key of the wrong type under a name + // the caller thought meant something else. + // + // Names the key type only. v1 has no signing modes, because it has no + // signing method: v0's `secp256k1_prehashed` was a mode wearing an + // algorithm's name, and it does not appear here. + string algorithm = 2; +} + +message GetKeyResponse { + // The derived private key: 32 raw bytes for both supported algorithms. + bytes key = 1; + // The corresponding public key. SEC1 compressed, 33 bytes, for secp256k1; + // 32 raw bytes for ed25519. This is the exact byte string the signature + // chain's first link commits to, so a relying party never has to re-derive + // it from `key` to check the chain. + bytes public_key = 2; + // Two links, in order: + // [0] the app root key's secp256k1 signature over the v1 key claim, which + // binds algorithm, domain and `public_key` -- see `docs/guest-api-v1.md` + // for the encoding and the verification steps; + // [1] the KMS root key's secp256k1 signature over the app root public key. + // + // Link [1] is produced by the KMS and is byte-identical to the one the + // unversioned surface returns. Link [0] is not: its claim encoding is new, + // and it is deliberately not one the v0 claim format can produce. + repeated bytes signature_chain = 3; +} + +// --------------------------------------------------------------------------- +// Attestation +// --------------------------------------------------------------------------- + +message AttestRequest { + // Up to 64 bytes of report data, zero-padded on the right to 64. + bytes report_data = 1; + // Also return the boot-time GPU attestation evidence in + // `AttestResponse.boottime_gpu_evidence`. This does not sample the GPU + // now and does not answer `report_data`; see that field. + bool include_boottime_gpu_evidence = 2; +} + +message AttestResponse { + // The versioned dstack attestation, always the MessagePack V1 schema + // (never the legacy SCALE form). See `docs/guest-api-v1.md`. + bytes attestation = 1; + + // Boot-time GPU attestation evidence. Empty unless the request set + // `include_boottime_gpu_evidence` and boot-time output exists, so absence is + // just the empty list rather than a sentinel value. + // + // Same bundle shape `AttestGpu` returns, so a consumer writes one parser for + // both. `format` is what separates them: this carries + // `nvidia-nvattest-boottime-json-v1`, the record written at boot, while + // `AttestGpu` carries `nvidia-nvattest-collect-evidence-json-v1`, collected + // on demand against a caller's nonce. Do not appraise one with the other's + // verifier. + // + // `evidence` is the exact UTF-8 bytes of the nvattest output as read from + // disk, byte for byte. That exactness is load-bearing: the only thing tying + // this evidence to the boot is sha256 over precisely these bytes, compared + // against the `evidence_sha256` field of the measured `gpu-attestation` + // event after replaying the runtime event log. Re-encoding or reformatting + // the JSON breaks the comparison. + // + // Not bound to `report_data`: nvattest ran at boot against its own nonce, so + // a fresh `report_data` says nothing about it. + // + // This is a historical statement about the boot, not a live one: it does not + // prove the GPU is still attached. Sampling the GPU at attestation time + // would not fix that -- an NVIDIA report binds the device and a nonce but + // not the TD the device is attached to, so a fresh report can be relayed + // from a genuine remote GPU. Only TDISP/TEE-IO device binding closes that. + repeated GpuEvidenceBundle boottime_gpu_evidence = 2; +} + +message AttestGpuRequest { + // Exactly 32 bytes of caller-chosen challenge, passed to the GPU verbatim. + // + // SPDM fixes the evidence nonce at 32 bytes, and dstack applies no transform + // so a caller can compare these bytes directly against the `eat_nonce` claim + // rather than reversing a hash. To bind a longer challenge, hash it + // yourself. + bytes nonce = 1; +} + +message AttestGpuResponse { + // Vendor-native evidence bundles. The caller must select a verifier using + // `vendor` and `format`, then verify the signature, certificate chain, + // measurements, and the nonce embedded in the evidence. + repeated GpuEvidenceBundle bundles = 1; +} + +// One vendor's GPU evidence. +// +// Shared by `AttestGpu` and `AttestResponse.boottime_gpu_evidence` so a +// consumer writes one parser for both. Dispatch on `(vendor, format)`; the two +// sources carry different `format` values because they answer different +// questions, and a verifier for one does not appraise the other. +message GpuEvidenceBundle { + // Stable GPU vendor identifier, for example `nvidia`, `amd`, or `intel`. + string vendor = 1; + // Vendor-specific evidence format and version. Known values: + // `nvidia-nvattest-collect-evidence-json-v1` fresh, from `AttestGpu` + // `nvidia-nvattest-boottime-json-v1` the record written at boot + string format = 2; + // Opaque vendor-native evidence bytes. Do not assume UTF-8 or JSON. + bytes evidence = 3; +} + +// --------------------------------------------------------------------------- +// Identity +// --------------------------------------------------------------------------- + +message InfoRequest {} + +// Identity and configuration. Not attestation. +// +// Everything here is what this application *is*: who it is, what it was +// configured with, and where it runs. Nothing here is evidence, and nothing +// here should be trusted on its own -- it arrives over a local socket with no +// quote behind it. +// +// That is why the measurement registers and the event log are absent. v0's +// `AppInfo.tcb_info` carried MRTD, RTMR0-3 and the event log as a JSON blob, +// which invited relying parties to read measurements out of an unattested +// response. Those values belong to `Attest`, whose `VersionedAttestation` +// carries them quote-backed. Ask `Attest` and verify, or do not use them. +// +// `mr_aggregated`, `os_image_hash` and `compose_hash` are the exception, and a +// deliberate one: they identify *which* application and image this is, which is +// the question `Info` answers. They are typed bytes here rather than hex +// strings inside a JSON blob, and each appears exactly once -- v0 returned all +// three both as top-level `AppInfo` fields and again, hex-encoded, inside +// `tcb_info`, two encodings of one fact and free to disagree. They are still +// unattested, and a relying party still confirms them against an attestation. +// +// `app_cert` is gone as well. It was a self-issued demo certificate the agent +// minted for a dashboard, proved nothing, and had no business on a key API. +message InfoResponse { + // Application -- what is deployed. + + // App ID. + bytes app_id = 1; + // App name, from app-compose. + string app_name = 2; + // Compose hash. Adjacent to the document it commits to, so a caller can see + // at a glance which field the hash covers. + bytes compose_hash = 3; + // The app-compose document, as a JSON document owned by the app-compose + // schema (`docs/normalized-app-compose.md`). + // + // These are the verbatim bytes that were deployed, and `compose_hash` is + // sha256 over exactly these bytes. Do not parse and re-serialize before + // hashing: key order, whitespace and unknown fields all change the digest, + // and that digest is what gets whitelisted on chain. + // + // Served directly rather than nested inside another JSON string, which is + // what v0 did via `tcb_info`. + string app_compose = 4; + + // Instance -- which running copy. + + // App instance ID. + bytes instance_id = 5; + + // Platform -- what it runs on. + + // Device ID. Identifies the host machine, not this instance. + bytes device_id = 6; + // OS image hash. + bytes os_image_hash = 7; + // Aggregated measurement register value. + bytes mr_aggregated = 8; + // The VM's hardware configuration, as a JSON document produced and owned by + // the VMM. The guest agent passes it through unparsed. + string vm_config = 9; + // The key provider that supplied this app's keys, as a JSON document owned + // by dstack-util (`{"name": ..., "id": ...}`), passed through unparsed. + string key_provider_info = 10; + // Cloud provider sys_vendor, for example "Google". + string cloud_vendor = 11; + // Cloud provider product_name, for example "Google Compute Engine". + string cloud_product = 12; +} + +message VersionRequest {} + +message VersionResponse { + // dstack version. + string version = 1; + // Git revision. + string rev = 2; +} + +// --------------------------------------------------------------------------- +// The external surface +// --------------------------------------------------------------------------- + +// The versioned external guest agent service. +// +// Served on the external listener at `/prpc/v1`, alongside the closed +// unversioned `Worker` at `/prpc`. +// +// Anyone who can route to the CVM can call this, so nothing here returns key +// material, and nothing here lets a caller choose what gets signed or attested. +// +// There is deliberately no `AttestAppKey`. v0's version attests the key its own +// KDF derives at path `vms` with purpose `signing`, and no v1 `GetKey(domain, +// algorithm)` can return that key -- so a pure-v1 application could be handed +// an attestation of a public key whose private half it has no way to obtain. +// A v1 application attests its own key instead: derive it at `/v1/GetKey`, +// commit the public key into `report_data`, and call `/v1/Attest`. The +// application then hands that attestation to its relying parties. Legacy flows +// keep using the frozen `Worker.GetAttestationForAppKey`. +service Worker { + // Return this application's identity and configuration. + // + // The same `InfoResponse` the internal surface returns, minus what the app + // asked to keep private: unless the app-compose sets `public_tcbinfo`, the + // three document fields (`app_compose`, `vm_config`, `key_provider_info`) + // come back empty. Identity and the measurement hashes are always present, + // which is what the unversioned `Worker.Info` also did. + rpc Info(InfoRequest) returns (InfoResponse) {} + + // Return the guest agent version. + rpc Version(VersionRequest) returns (VersionResponse) {} + + // Report whether the app is serving. Polled by the gateway to decide whether + // this instance should be in its app's load-balancing rotation. + // + // Answers from a cache the agent refreshes on its own timer, so the cost of + // one call is a lock and a clone however many gateway nodes are polling. + // + // Only instances that asked for gating (`RegisterCvmRequest.health_check`) + // are ever polled. That flag, not this method's absence, is what protects an + // older image: the gateway counts *any* failed poll as unhealthy, so probing + // to discover support would blackhole every guest agent that predates this. + rpc Health(HealthRequest) returns (HealthResponse) {} +} + +message HealthRequest {} + +// One container that is not reporting healthy. +message ContainerHealth { + // Container name, without the leading slash Docker prepends. + string name = 1; + // Docker health state: "starting" or "unhealthy". A healthy container is + // never listed. A string rather than an enum so it stays readable as JSON. + string status = 2; +} + +// Aggregate application health, as the agent last determined it. +message HealthResponse { + // False when the app's own health file says so or has gone stale, or -- when + // no health file was declared -- when any container that declares a Compose + // `healthcheck` is not running and healthy. + // + // True, unconditionally and with an empty `unhealthy`, for an app that never + // opted into health gating: it registered as "do not poll me", so no gateway + // asks, and anything that asks anyway gets the answer the gateway would have + // assumed. "Healthy" there means "nobody asked me to know", not "checked and + // fine" -- `RegisterCvmRequest.health_check` is what separates the two, and + // this method cannot report the difference. + bool healthy = 1; + // The containers that made `healthy` false, so an operator reading gateway + // logs can tell which one is holding the instance out of rotation. Purely + // diagnostic -- the gateway routes on `healthy` alone. + repeated ContainerHealth unhealthy = 2; + // Set when the agent could not see the app at all for several refreshes in + // a row (container runtime unreachable, permission denied). `healthy` is + // false in that case: an agent that cannot see the app is in no position to + // vouch for it. + // + // Reported in the response rather than as an RPC error on purpose. The + // gateway's health poller counts any failed poll as a failed poll; it does + // not read the status code or the body. A verdict of "the agent cannot see + // the app" raised as an RPC error would therefore be filed alongside an + // agent that is unreachable or has no such method, and the reason would + // never reach the operator. Carried here alongside `healthy` false, it does. + string error = 3; +} diff --git a/dstack/guest-agent/rpc/src/generated.rs b/dstack/guest-agent/rpc/src/generated.rs index 28748082f..352061b2e 100644 --- a/dstack/guest-agent/rpc/src/generated.rs +++ b/dstack/guest-agent/rpc/src/generated.rs @@ -3,4 +3,23 @@ pub const FILE_DESCRIPTOR_SET: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/file_descriptor_set.bin")); -include!(concat!(env!("OUT_DIR"), "/dstack_guest.rs")); +/// The frozen v0 surface, closed at exactly what dstack v0.5.11 shipped. +/// +/// Served at `/v0` and, for pre-0.6 clients, at the unversioned paths it has +/// always had. Nothing new goes here. +pub mod v0 { + include!(concat!(env!("OUT_DIR"), "/dstack_guest.rs")); +} + +/// The `dstack.guest.v1` package: the current surface. +/// +/// Each version gets its own module because both packages define types with the +/// same names on purpose -- `GpuEvidenceBundle`, and the services themselves -- +/// and the two surfaces must stay independently evolvable. A caller names the +/// surface it wants: `dstack_guest_agent_rpc::v0::AppInfo` is the frozen one, +/// `dstack_guest_agent_rpc::v1::InfoResponse` is the current one. Nothing is +/// re-exported at the crate root, so no import can be ambiguous about which +/// contract it speaks. +pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/dstack.guest.v1.rs")); +} diff --git a/dstack/guest-agent/rpc/tests/frozen_surface.rs b/dstack/guest-agent/rpc/tests/frozen_surface.rs new file mode 100644 index 000000000..6f9c3b95c --- /dev/null +++ b/dstack/guest-agent/rpc/tests/frozen_surface.rs @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! A structural freeze on the v0.5.11 wire surface. +//! +//! `DstackGuest`, `Worker` and `Tappd` are closed: they are exactly what +//! v0.5.11 shipped, and every new capability belongs to `dstack.guest.v1`. +//! Reviewing that by eye is how three never-released methods and a handful of +//! fields accumulated on them between v0.5.11 and 0.6.0 in the first place. +//! +//! So this pins a digest of each frozen service's descriptor: its methods, and +//! the full field list of every message reachable from them. A *wire-compatible* +//! addition -- a new optional field, a new method -- changes the digest and +//! fails here, which is the point. Nothing else in the tree would notice. +//! +//! If this test fails, the fix is almost always to move the addition to +//! `agent_rpc_v1.proto`, not to update the digest. + +use prost::Message; +use prost_types::{ + DescriptorProto, FileDescriptorProto, FileDescriptorSet, ServiceDescriptorProto, +}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; + +fn descriptor_set() -> FileDescriptorSet { + FileDescriptorSet::decode(dstack_guest_agent_rpc::FILE_DESCRIPTOR_SET) + .expect("the embedded descriptor set must decode") +} + +fn frozen_file(set: &FileDescriptorSet) -> &FileDescriptorProto { + set.file + .iter() + .find(|file| file.package() == "dstack_guest") + .expect("the frozen package must be present") +} + +fn message<'a>(file: &'a FileDescriptorProto, name: &str) -> &'a DescriptorProto { + file.message_type + .iter() + .find(|message| message.name() == name) + .unwrap_or_else(|| panic!("frozen message {name} vanished")) +} + +/// Render a message as one line per field: number, name, type, label. +/// +/// Type and label are included so that changing a field's type or making it +/// repeated is caught, not just adding or removing one. Reserved ranges are +/// included too: dropping a `reserved` frees a number for silent reuse. +fn describe_message(file: &FileDescriptorProto, name: &str) -> String { + let message = message(file, name); + let mut out = format!("message {name}\n"); + let mut fields: Vec<String> = message + .field + .iter() + .map(|field| { + format!( + " {} {} type={:?} label={:?}\n", + field.number(), + field.name(), + field.r#type(), + field.label() + ) + }) + .collect(); + fields.sort(); + out.extend(fields); + for range in &message.reserved_range { + out.push_str(&format!(" reserved {}..{}\n", range.start(), range.end())); + } + for name in &message.reserved_name { + out.push_str(&format!(" reserved-name {name}\n")); + } + out +} + +/// Render a service as its methods plus every message it can reach. +fn describe_service(file: &FileDescriptorProto, service: &ServiceDescriptorProto) -> String { + let mut out = format!("service {}\n", service.name()); + let mut reachable = BTreeSet::new(); + for method in &service.method { + out.push_str(&format!( + " rpc {}({}) returns ({})\n", + method.name(), + method.input_type(), + method.output_type() + )); + for type_name in [method.input_type(), method.output_type()] { + // `.dstack_guest.Foo` -> `Foo`; google.protobuf.Empty is not ours. + if let Some(local) = type_name.strip_prefix(".dstack_guest.") { + reachable.insert(local.to_string()); + } + } + } + // One level of nesting is enough for this surface: the only message-typed + // field on it is `HealthResponse.unhealthy`, which v1 owns now. + for name in reachable.clone() { + for field in &message(file, &name).field { + if let Some(local) = field.type_name().strip_prefix(".dstack_guest.") { + reachable.insert(local.to_string()); + } + } + } + for name in &reachable { + out.push_str(&describe_message(file, name)); + } + out +} + +fn digest(shape: &str) -> String { + hex::encode(Sha256::digest(shape.as_bytes())) +} + +/// The three frozen services, pinned. +/// +/// Regenerate a digest only when you have confirmed against +/// `git show v0.5.11:guest-agent/rpc/proto/agent_rpc.proto` that the change is +/// comment-only or a `reserved` addition. +#[test] +fn the_frozen_services_match_their_pinned_shape() { + let set = descriptor_set(); + let file = frozen_file(&set); + let expected = [ + ( + "DstackGuest", + "aa6b5814627a26284b12c180acb0eb13f91c3e57ce00952734fb358a729762f3", + ), + ( + "Worker", + "e5e88ddbba3e9acd2ac68c6f5e3c99e394c1842be3d01395dd214972ada5bdc1", + ), + ( + "Tappd", + "65c2e3b49f0ffbdcda4712f4b87b2ca4be386205e57e7956ddc56bec7616cffd", + ), + ]; + for (name, want) in expected { + let service = file + .service + .iter() + .find(|service| service.name() == name) + .unwrap_or_else(|| panic!("frozen service {name} vanished")); + let shape = describe_service(file, service); + assert_eq!( + digest(&shape), + want, + "the frozen {name} surface changed:\n{shape}" + ); + } +} + +/// The method lists, spelled out, so a failure above is readable without +/// reaching for the descriptor dump. +#[test] +fn the_frozen_services_expose_the_v0_5_11_methods() { + let set = descriptor_set(); + let file = frozen_file(&set); + let methods = |name: &str| -> Vec<String> { + file.service + .iter() + .find(|service| service.name() == name) + .unwrap_or_else(|| panic!("frozen service {name} vanished")) + .method + .iter() + .map(|method| method.name().to_string()) + .collect() + }; + assert_eq!( + methods("DstackGuest"), + [ + "GetTlsKey", + "GetKey", + "GetQuote", + "Attest", + "EmitEvent", + "Info", + "Sign", + "Verify", + "Version" + ] + ); + assert_eq!( + methods("Worker"), + ["Info", "Version", "GetAttestationForAppKey"] + ); + assert_eq!( + methods("Tappd"), + [ + "DeriveKey", + "DeriveK256Key", + "TdxQuote", + "RawQuote", + "Info", + "Version" + ] + ); +} diff --git a/dstack/guest-agent/src/backend.rs b/dstack/guest-agent/src/backend.rs index a87b6c6e2..fcdf471bb 100644 --- a/dstack/guest-agent/src/backend.rs +++ b/dstack/guest-agent/src/backend.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::{Context, Result}; -use dstack_guest_agent_rpc::{AttestResponse, GetQuoteResponse}; +use dstack_guest_agent_rpc::v0::GetQuoteResponse; use ra_tls::attestation::Attestation; use ra_tls::attestation::{QuoteContentType, VersionedAttestation}; @@ -11,7 +11,9 @@ pub trait PlatformBackend: Send + Sync { fn attestation_for_info(&self) -> Result<VersionedAttestation>; fn certificate_attestation(&self, pubkey: &[u8]) -> Result<VersionedAttestation>; fn quote_response(&self, report_data: [u8; 64], vm_config: &str) -> Result<GetQuoteResponse>; - fn attest_response(&self, report_data: [u8; 64]) -> Result<AttestResponse>; + /// Attest the CVM itself: the attestation `Attest` returns, with digest + /// preimages filled in. Encoding it is the RPC layer's job. + fn attest_cvm(&self, report_data: [u8; 64]) -> Result<VersionedAttestation>; } #[derive(Debug, Default)] @@ -33,31 +35,21 @@ impl PlatformBackend for RealPlatform { fn quote_response(&self, report_data: [u8; 64], vm_config: &str) -> Result<GetQuoteResponse> { let attestation = Attestation::quote(&report_data).context("Failed to get quote")?; - let tdx_quote = attestation.get_tdx_quote_bytes(); - let tdx_event_log = attestation.get_tdx_event_log_string_for_config(vm_config); - // TDX callers already have quote + event_log. Only non-TDX platforms - // need the platform-adaptive versioned attestation payload. - let versioned = if tdx_quote.is_some() { - Vec::new() - } else { - attestation - .into_versioned() - .to_bytes() - .context("Failed to encode versioned attestation")? - }; + let quote = attestation + .get_tdx_quote_bytes() + .context("GetQuote is Intel TDX only, use Attest on this platform")?; Ok(GetQuoteResponse { - quote: tdx_quote.unwrap_or_default(), - event_log: tdx_event_log.unwrap_or_default(), + quote, + event_log: attestation.get_tdx_event_log_string().unwrap_or_default(), report_data: report_data.to_vec(), vm_config: vm_config.to_string(), - attestation: versioned, }) } - fn attest_response(&self, report_data: [u8; 64]) -> Result<AttestResponse> { - let attestation = Attestation::quote(&report_data).context("Failed to get attestation")?; - Ok(AttestResponse { - attestation: attestation.into_versioned().to_bytes()?, - }) + fn attest_cvm(&self, report_data: [u8; 64]) -> Result<VersionedAttestation> { + let mut attestation = + Attestation::quote(&report_data).context("Failed to get attestation")?; + attestation.fill_event_preimages(); + Ok(attestation.into_versioned()) } } diff --git a/dstack/guest-agent/src/config.rs b/dstack/guest-agent/src/config.rs index 7f94185d7..fb816f2f1 100644 --- a/dstack/guest-agent/src/config.rs +++ b/dstack/guest-agent/src/config.rs @@ -48,8 +48,6 @@ pub struct Config { #[serde(deserialize_with = "deserialize_app_compose", flatten)] pub app_compose: AppComposeWrapper, pub sys_config_file: PathBuf, - #[serde(default)] - pub pccs_url: Option<String>, pub data_disks: HashSet<PathBuf>, } @@ -72,3 +70,146 @@ where raw: content, }) } + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + use tempfile::TempDir; + + const DEFAULTS: &str = r#" +[default.core] +keys_file = "/embedded/keys.json" +compose_file = "/embedded/compose.json" +sys_config_file = "/embedded/sys-config.json" +data_disks = ["/"] +"#; + + fn minimal_compose(extra: &str) -> String { + format!( + r#"{{ + "manifest_version": 2, + "name": "config-entry-test", + "runner": "docker-compose", + "docker_compose_file": "services: {{}}"{extra} +}}"# + ) + } + + fn leaf_config(directory: &Path, compose: &Path, extra: &str) -> PathBuf { + let leaf = directory.join("guest.toml"); + fs::write( + &leaf, + format!( + r#" +[default.core] +keys_file = "{directory}/keys.json" +compose_file = "{compose}" +sys_config_file = "{directory}/sys-config.json" +data_disks = ["/", "{directory}/data"] +{extra} +"#, + directory = directory.display(), + compose = compose.display(), + ), + ) + .unwrap(); + leaf + } + + fn extract(defaults: &str, leaf: &Path) -> Result<Config, Box<figment::Error>> { + load_config_figment_with_default(defaults, leaf.to_str()) + .extract_inner("core") + .map_err(Box::new) + } + + #[test] + fn explicit_leaf_overrides_embedded_defaults() { + let temporary = TempDir::new().unwrap(); + let compose = temporary.path().join("compose.json"); + fs::write(&compose, minimal_compose("")).unwrap(); + let leaf = leaf_config(temporary.path(), &compose, ""); + + let config = extract(DEFAULTS, &leaf).unwrap(); + + assert_eq!( + config.keys_file, + temporary.path().join("keys.json").display().to_string() + ); + assert_eq!( + config.sys_config_file, + temporary.path().join("sys-config.json") + ); + assert!(config.data_disks.contains(&PathBuf::from("/"))); + assert!(config.data_disks.contains(&temporary.path().join("data"))); + } + + #[test] + fn compose_raw_bytes_and_unknown_fields_are_preserved() { + let temporary = TempDir::new().unwrap(); + let compose = temporary.path().join("compose.json"); + let raw = minimal_compose(",\n \"future_optional_field\": {\"enabled\": true}"); + fs::write(&compose, &raw).unwrap(); + let leaf = leaf_config(temporary.path(), &compose, ""); + + let config = extract(DEFAULTS, &leaf).unwrap(); + + assert_eq!(config.app_compose.raw, raw); + assert_eq!(config.app_compose.name, "config-entry-test"); + } + + #[test] + fn absent_optional_compose_fields_use_documented_defaults() { + let temporary = TempDir::new().unwrap(); + let compose = temporary.path().join("compose.json"); + fs::write(&compose, minimal_compose("")).unwrap(); + let leaf = leaf_config(temporary.path(), &compose, ""); + + let config = extract(DEFAULTS, &leaf).unwrap(); + + assert!(!config.app_compose.public_logs); + assert!(!config.app_compose.public_sysinfo); + assert!(config.app_compose.public_tcbinfo); + assert!(!config.app_compose.kms_enabled); + assert!(!config.app_compose.gateway_enabled); + assert!(config.app_compose.secure_time); + assert_eq!(config.app_compose.swap_size, 0); + } + + #[test] + fn missing_compose_file_fails_before_state_construction() { + let temporary = TempDir::new().unwrap(); + let missing = temporary.path().join("missing-compose.json"); + let leaf = leaf_config(temporary.path(), &missing, ""); + + let error = extract(DEFAULTS, &leaf).unwrap_err().to_string(); + + assert!(error.contains("Failed to read compose file"), "{error}"); + } + + #[test] + fn malformed_or_required_field_missing_compose_fails_closed() { + let temporary = TempDir::new().unwrap(); + let compose = temporary.path().join("compose.json"); + let leaf = leaf_config(temporary.path(), &compose, ""); + + fs::write(&compose, "{not-json").unwrap(); + let malformed = extract(DEFAULTS, &leaf).unwrap_err().to_string(); + assert!( + malformed.contains("Failed to parse compose file"), + "{malformed}" + ); + + fs::write( + &compose, + r#"{"manifest_version":2,"runner":"docker-compose"}"#, + ) + .unwrap(); + let missing = extract(DEFAULTS, &leaf).unwrap_err().to_string(); + assert!( + missing.contains("Failed to parse compose file"), + "{missing}" + ); + assert!(missing.contains("missing field `name`"), "{missing}"); + } +} diff --git a/dstack/guest-agent/src/container_health.rs b/dstack/guest-agent/src/container_health.rs new file mode 100644 index 000000000..9c4c69ed3 --- /dev/null +++ b/dstack/guest-agent/src/container_health.rs @@ -0,0 +1,899 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Aggregate container health, as polled by the gateway. +//! +//! The gateway uses this to decide whether this instance belongs in its app's +//! load-balancing rotation. Registration happens during boot, long before +//! `app-compose.service` has pulled an image or started a container, so +//! "registered" on its own is a poor proxy for "can serve a request". +//! +//! Both container runners are covered. Docker is queried over its socket; +//! nerdctl is queried by running the CLI, because its state lives in containerd +//! rather than behind a Docker-compatible socket. The two report the same four +//! health strings -- nerdctl's inspect output is generated from its +//! `dockercompat` types -- so the rule itself is shared. +//! +//! Only the app's own Compose project is judged. Container state lives on the +//! persistent data disk (`/var/lib/{docker,containerd,nerdctl}` are rbind +//! mounted there by `dstack-prepare.sh`), so it outlives reboots and app +//! upgrades: without the filter, one exited container left behind by a previous +//! deployment under a different project name would hold the instance out of +//! rotation forever, and nothing cleans it up -- `--remove-orphans` only ever +//! touches the project it was invoked for. +//! +//! What they do not agree on is what happens to that string once the container +//! stops, which is why the run state is read alongside it. See [`judge`]. + +use std::collections::HashMap; +use std::time::Duration; + +use anyhow::{bail, Context, Result}; +use bollard::container::ListContainersOptions; +use bollard::Docker; +use futures::{StreamExt, TryStreamExt}; +use serde::Deserialize; +use tokio::process::Command; +use tracing::debug; + +use dstack_guest_agent_rpc::v1::ContainerHealth; + +use crate::health::Verdict; + +/// How many containers to inspect at once on the docker path. +/// +/// Bounded rather than unbounded: the daemon is shared with whatever the app is +/// doing, and a burst of one request per container is a poor way to ask a +/// process that may already be the reason health is being checked. nerdctl needs +/// no equivalent -- it inspects the whole project in one call. +const INSPECT_CONCURRENCY: usize = 8; + +/// The label Compose stamps on every container it starts. +/// +/// This is the only project label both runners agree on. Measured on docker +/// compose 5.1.4 and nerdctl 2.3.5: docker also writes +/// `com.docker.compose.project.working_dir` and `.config_files`, nerdctl writes +/// neither -- it sets exactly `project`, `service` and `config-hash`. Filtering +/// on a docker-only label would return an empty list under nerdctl, which +/// `judge` reads as "nothing has started yet", i.e. permanently unhealthy on +/// the runner the guest image actually uses. +const COMPOSE_PROJECT_LABEL: &str = "com.docker.compose.project"; + +/// Where `app-compose.sh` records the namespace and project it used. +/// +/// Read rather than derived. Compose resolves the project name from +/// `COMPOSE_PROJECT_NAME` before the compose file's top-level `name:` -- +/// verified on docker compose 5.1.4 and nerdctl 2.3.5, where both honour the +/// env var -- and the app supplies that env through `.decrypted-env`. Deriving +/// the name from the compose file therefore gets it wrong exactly when an app +/// sets it, and the containerd namespace is not in the compose file at all. +/// Both mistakes look identical from here: an empty container list, which +/// `judge` reads as "nothing started yet", i.e. permanently unhealthy with a +/// message that says the opposite of what happened. +/// +/// On tmpfs, and written by `app-compose.sh` before it starts anything, so its +/// absence means the app has not been started yet -- which is the truth, and is +/// what gets reported. +const COMPOSE_RUNTIME_FILE: &str = "/run/dstack/app-compose-runtime.json"; + +/// How long a single container-runtime call may take. +/// +/// Without this a wedged containerd stalls the refresh loop indefinitely; with +/// it, plus `kill_on_drop`, an abandoned call also reclaims its child process. +const RUNTIME_TIMEOUT: Duration = Duration::from_secs(5); + +/// One container as the runtime described it. +#[derive(Debug, PartialEq)] +struct Observed { + name: String, + /// `State.Health.Status`. Absent when the container declares no + /// healthcheck: both runtimes populate this field only for containers that + /// do. + health: Option<String>, + /// `State.Status`: `running`, `exited`, `paused`, `restarting`, ... Absent + /// when the runtime did not report one. + state: Option<String>, +} + +/// Judge the app's containers, dispatching on the runner it was started with. +/// +/// Only containers that declare a Compose `healthcheck` are judged. Both +/// runtimes populate a health status exclusively for those, so its absence is +/// the app saying "I have not told you how to test this one" rather than a +/// pass. That keeps this from failing an app for a one-shot init container +/// that exited cleanly. +pub(crate) async fn collect(runner: &str) -> Result<Verdict> { + match runner { + "docker-compose" | "nerdctl-compose" => {} + // The `bash` runner runs a script, not containers. There is nothing to + // inspect and never will be, so it must not sit at "not started yet". + // Nothing to judge, so nothing to hold traffic back for. + _ => return Ok(Verdict::healthy()), + } + let Some(runtime) = ComposeRuntime::read().await? else { + // `app-compose.sh` writes this before it starts anything, so its + // absence is the boot window this feature exists to keep traffic out + // of -- not an error, and not a pass. + return Ok(judge(vec![])); + }; + match runner { + "docker-compose" => collect_docker(&runtime.project).await, + _ => collect_nerdctl(&runtime).await, + } +} + +/// What `app-compose.sh` recorded about the deployment it started. +#[derive(Debug, Deserialize, PartialEq, Eq)] +pub(crate) struct ComposeRuntime { + /// The containerd namespace nerdctl was run under. + pub namespace: String, + /// The Compose project name, as Compose itself resolved it. + pub project: String, + /// The snapshotter nerdctl was run with. + /// + /// Every nerdctl call in `app-compose.sh` passes `--snapshotter`, so these + /// ones do too. Whether `ps` and `inspect` actually need it for a stargz + /// app is not something this could be tested against here, and matching the + /// runner costs nothing -- for the default `overlayfs` the flag is a no-op, + /// and for anything else guessing wrong is a permanently unhealthy app. + #[serde(default)] + pub snapshotter: Option<String>, +} + +impl ComposeRuntime { + async fn read() -> Result<Option<Self>> { + Self::read_from(COMPOSE_RUNTIME_FILE).await + } + + async fn read_from(path: &str) -> Result<Option<Self>> { + let raw = match tokio::fs::read_to_string(path).await { + Ok(raw) => raw, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(err).with_context(|| format!("failed to read {path}")), + }; + let runtime: Self = + serde_json::from_str(&raw).with_context(|| format!("failed to parse {path}"))?; + if runtime.project.is_empty() { + bail!("{path} names no compose project"); + } + Ok(Some(runtime)) + } +} + +/// The label filter both runners are queried with, as `key=value`. +fn compose_filter(project: &str) -> String { + format!("{COMPOSE_PROJECT_LABEL}={project}") +} + +async fn collect_docker(project: &str) -> Result<Verdict> { + let docker = Docker::connect_with_defaults().context("failed to connect to docker")?; + let filter = compose_filter(project); + let mut filters = HashMap::new(); + filters.insert("label", vec![filter.as_str()]); + let containers = docker + .list_containers(Some(ListContainersOptions { + all: true, + filters, + ..Default::default() + })) + .await + .context("failed to list containers")?; + + // Concurrently, because the whole refresh has ten seconds and this is one + // round trip per container. Serially, a compose project with a few dozen + // services can spend that budget on the daemon alone, and a refresh that + // times out is reported as "could not tell" -- three of those in a row and + // the app is called unhealthy on the strength of the agent being slow. + let observed = futures::stream::iter(containers) + .map(|summary| { + let docker = &docker; + async move { + let id = summary.id.as_deref()?; + let details = match docker.inspect_container(id, None).await { + Ok(details) => details, + // The container went away between the list and the inspect. + // `docker compose up` recreates containers on every + // redeploy and `app-compose.sh` prunes right after, so this + // window is raced routinely. Judging the rest is strictly + // better than failing the whole report and dropping the + // instance out of rotation. + Err(bollard::errors::Error::DockerResponseServerError { + status_code: 404, + .. + }) => { + debug!("container {id} disappeared between list and inspect"); + return None; + } + Err(err) => { + return Some(Err(anyhow::Error::new(err) + .context(format!("failed to inspect container {id}")))) + } + }; + let state = details.state; + let health = state + .as_ref() + .and_then(|state| state.health.as_ref()) + .and_then(|health| health.status) + .map(|status| status.to_string()); + let run_state = state + .as_ref() + .and_then(|state| state.status) + .map(|status| status.to_string()); + Some(Ok(Observed { + name: container_name(summary.names.as_deref(), id), + health, + state: run_state, + })) + } + }) + .buffer_unordered(INSPECT_CONCURRENCY) + .filter_map(|result| async move { result }) + .try_collect::<Vec<_>>() + .await?; + Ok(judge(observed)) +} + +async fn collect_nerdctl(runtime: &ComposeRuntime) -> Result<Verdict> { + let filter = format!("label={}", compose_filter(&runtime.project)); + let ids = nerdctl(runtime, &["ps", "-a", "-q", "--filter", &filter]) + .await + .context("failed to list nerdctl containers")?; + let ids = ids + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .collect::<Vec<_>>(); + if ids.is_empty() { + return Ok(judge(vec![])); + } + Ok(judge(inspect_nerdctl(runtime, &ids).await?)) +} + +/// Inspect every id, tolerating the ones that vanished under us. +/// +/// `nerdctl inspect a b c` exits non-zero if *any* id is gone, and -- unlike +/// docker, which still writes the entries it did find -- it writes **nothing** +/// to stdout. Measured on 2.3.5 with one bogus id in a batch of three: exit 1, +/// 0 bytes of stdout, `level=fatal msg="1 errors: [no such object ...]"` on +/// stderr, regardless of where in the batch the bogus id sits. +/// +/// So one container recreated between `ps` and `inspect` -- which every +/// redeploy does, and `app-compose.sh`'s prune right after start makes routine +/// -- would otherwise fail the entire report and drop the instance out of +/// rotation. The batch stays the fast path; only when it fails do we pay for +/// one call per container and skip the ones that no longer exist. +async fn inspect_nerdctl(runtime: &ComposeRuntime, ids: &[&str]) -> Result<Vec<Observed>> { + let mut args = vec!["inspect"]; + args.extend_from_slice(ids); + match nerdctl(runtime, &args).await { + Ok(raw) => Ok(parse_nerdctl(&raw)?), + Err(err) => { + debug!("batch inspect failed, falling back to one call per container: {err:#}"); + let mut observed = Vec::with_capacity(ids.len()); + for id in ids { + match nerdctl(runtime, &["inspect", id]).await { + Ok(raw) => observed.extend(parse_nerdctl(&raw)?), + Err(err) => debug!("skipping container {id}: {err:#}"), + } + } + // Every single inspect failing too means the runtime itself is the + // problem, not a raced container. Report the original error rather + // than an empty list, which `judge` would read as "nothing started + // yet". + if observed.is_empty() { + return Err(err); + } + Ok(observed) + } + } +} + +fn parse_nerdctl(raw: &str) -> Result<Vec<Observed>> { + let containers: Vec<NerdctlContainer> = + serde_json::from_str(raw).context("failed to parse nerdctl inspect output")?; + Ok(containers.into_iter().map(observe_nerdctl).collect()) +} + +async fn nerdctl(runtime: &ComposeRuntime, args: &[&str]) -> Result<String> { + let mut command = Command::new("nerdctl"); + command.arg("--namespace").arg(&runtime.namespace); + if let Some(snapshotter) = &runtime.snapshotter { + command.arg("--snapshotter").arg(snapshotter); + } + let run = command + .args(args) + // Without this, abandoning the future on timeout leaves the child + // running: `tokio::process` does not kill it on drop by default. + .kill_on_drop(true) + .output(); + let output = match tokio::time::timeout(RUNTIME_TIMEOUT, run).await { + Ok(output) => output.context("failed to run nerdctl")?, + Err(_) => bail!("nerdctl {} timed out after {RUNTIME_TIMEOUT:?}", args[0]), + }; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + bail!("nerdctl {} failed: {}", args.join(" "), stderr.trim()); + } + String::from_utf8(output.stdout).context("nerdctl produced non-utf8 output") +} + +/// The slice of `nerdctl inspect` output this needs. Generated from nerdctl's +/// `dockercompat` types, so the shape and the status strings match Docker's. +#[derive(Deserialize)] +struct NerdctlContainer { + #[serde(rename = "Id", default)] + id: String, + #[serde(rename = "Name", default)] + name: String, + #[serde(rename = "State", default)] + state: Option<NerdctlState>, +} + +#[derive(Deserialize)] +struct NerdctlState { + #[serde(rename = "Status", default)] + status: String, + #[serde(rename = "Health", default)] + health: Option<NerdctlHealth>, +} + +/// Reshape one `nerdctl inspect` entry into what [`judge`] reads. +fn observe_nerdctl(container: NerdctlContainer) -> Observed { + let name = if container.name.is_empty() { + container.id + } else { + container.name + }; + let (health, state) = match container.state { + Some(state) => ( + state.health.map(|health| health.status), + (!state.status.is_empty()).then_some(state.status), + ), + None => (None, None), + }; + Observed { + name: name.trim_start_matches('/').to_string(), + health, + state, + } +} + +#[derive(Deserialize)] +struct NerdctlHealth { + #[serde(rename = "Status", default)] + status: String, +} + +/// Turn what the runtime reported into a verdict. +/// +/// Split from the collectors so the rule itself can be tested without a +/// container runtime to talk to, and so both runners share one definition of +/// what "healthy" means. +/// +/// `starting` counts as not healthy: that state is exactly the boot window this +/// is meant to keep traffic out of. So does finding no container at all, which +/// means Compose has not yet got as far as creating one. +/// +/// A container that declares a healthcheck and is not running is not healthy +/// either, whatever its last recorded verdict says. That verdict cannot be +/// trusted across a stop, because the two runtimes disagree about it: +/// +/// - Docker's daemon re-marks the container `unhealthy` when it leaves the +/// running state (verified on 29.5.3 for `stop`, `kill`, `pause`, and a +/// container exiting on its own). +/// - nerdctl leaves the last result in place. Its `container healthcheck` +/// returns early with "container is not running" *without* writing a new +/// state, and `nerdctl inspect` renders `State.Health` out of the stored +/// state regardless of run state. Verified on 2.3.5: a container that passed +/// its check and then exited -- by `nerdctl stop` or by crashing -- still +/// reports `"Status": "healthy"` with `"Running": false`. +/// +/// nerdctl is what the guest image runs, so without this an app whose container +/// crashed after one passing check keeps its instance in the rotation +/// indefinitely: exactly the state this feature exists to detect. +/// +/// Containers that declare no healthcheck stay unjudged even when they have +/// exited. A one-shot init container that ran and stopped is normal, and the +/// app has not said how to tell the difference. But if *no* container is +/// judged, the app asked to be gated on an answer nothing can produce, and +/// saying "healthy" to that is the silent no-op this feature exists to remove. +/// That is checked here rather than by scanning the compose file at deploy +/// time, because this is the only place that sees what the runtime actually +/// did: a `HEALTHCHECK` in a Dockerfile never appears in the compose file, a +/// `healthcheck:` merged in from a YAML anchor is invisible to a key scan, and +/// `healthcheck: {disable: true}` is present in the file and produces nothing +/// here. +fn judge(containers: Vec<Observed>) -> Verdict { + // No container exists yet. Registration happens in + // `dstack-prepare.service` and `app-compose.service` is ordered after it, + // so this is the window where the image may still be pulling -- exactly + // what the gateway must keep traffic out of. A runtime answering while + // having nothing to show is not the same as the app being fine. + if containers.is_empty() { + return Verdict::unhealthy("<compose>", "no container has been created yet".to_string()); + } + + let mut unhealthy = Vec::new(); + let mut judged = 0usize; + for container in containers { + let Observed { + name, + health, + state, + } = container; + match health.as_deref() { + // No healthcheck declared, or none reported yet. + None | Some("") | Some("none") => continue, + Some(health) if !is_running(state.as_deref()) => unhealthy.push(ContainerHealth { + name, + status: format!( + "{} (last health check: {health})", + state.as_deref().unwrap_or("not running") + ), + }), + Some("healthy") => {} + // "starting", "unhealthy", and anything a future runtime version + // invents. Unknown states hold traffic back rather than pass. + Some(other) => unhealthy.push(ContainerHealth { + name, + status: other.to_string(), + }), + } + judged += 1; + } + + if judged == 0 { + // Containers exist, and not one of them can be judged. Reported rather + // than passed: this app asked for its traffic to be gated on an answer, + // and answering "healthy" unconditionally is worse than saying so -- + // the operator would see a feature that is switched on and doing + // nothing, with no signal but the absence of an effect. + return Verdict::unhealthy( + "<compose>", + "no container declares a healthcheck, so nothing can be judged; add one, or set \ + requirements.health_status_file to report health directly" + .to_string(), + ); + } + + // By name, because the docker collector inspects concurrently and hands + // them back in completion order. The gateway names the first eight and + // counts the rest, so an unstable order would rewrite that log line on + // every poll for a project with more than eight failing containers -- + // making a steady failure read as a changing one. + unhealthy.sort_by(|a, b| a.name.cmp(&b.name)); + Verdict::from_containers(unhealthy) +} + +/// Whether the runtime says this container is running. +/// +/// A container the runtime gave no state for is judged on its health alone: a +/// missing field is not evidence that it stopped, and inventing a failure from +/// one would take an app out of rotation over a shape change in the runtime's +/// output. +fn is_running(state: Option<&str>) -> bool { + match state { + None => true, + Some(state) => state == "running", + } +} + +/// Docker returns names with a leading slash; strip it so log lines read the +/// way `docker ps` prints them. Falls back to the id when a container somehow +/// has no name. +fn container_name(names: Option<&[String]>, id: &str) -> String { + names + .and_then(|names| names.first()) + .map(|name| name.trim_start_matches('/').to_string()) + .unwrap_or_else(|| id.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A running container reporting `status`. + fn named(name: &str, status: Option<&str>) -> Observed { + Observed { + name: name.to_string(), + health: status.map(str::to_string), + state: Some("running".to_string()), + } + } + + /// A container that is no longer running, with whatever health verdict the + /// runtime left behind. + fn stopped(name: &str, state: &str, status: Option<&str>) -> Observed { + Observed { + name: name.to_string(), + health: status.map(str::to_string), + state: Some(state.to_string()), + } + } + + fn runtime_file(dir: &tempfile::TempDir, contents: &str) -> String { + let path = dir.path().join("app-compose-runtime.json"); + std::fs::write(&path, contents).expect("write"); + path.to_str().expect("utf8").to_string() + } + + /// The project name is whatever Compose resolved, not whatever the compose + /// file says: `COMPOSE_PROJECT_NAME` outranks the file's `name:`, and an + /// app supplies that env through `.decrypted-env`. + #[tokio::test] + async fn the_recorded_project_is_used_verbatim() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = runtime_file(&dir, r#"{"namespace":"dstack","project":"envwins"}"#); + let runtime = ComposeRuntime::read_from(&path) + .await + .expect("read") + .expect("present"); + assert_eq!(runtime.project, "envwins"); + assert_eq!(runtime.namespace, "dstack"); + } + + /// Every nerdctl call in `app-compose.sh` passes `--snapshotter`, so these + /// have to as well; assuming the default would be a guess that shows up as + /// a permanently unhealthy stargz app. + #[tokio::test] + async fn the_recorded_snapshotter_is_carried_through() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = runtime_file( + &dir, + r#"{"namespace":"dstack","project":"app","snapshotter":"stargz"}"#, + ); + let runtime = ComposeRuntime::read_from(&path) + .await + .expect("read") + .expect("present"); + assert_eq!(runtime.snapshotter.as_deref(), Some("stargz")); + } + + /// A runtime file written before the snapshotter was recorded still works; + /// nerdctl then uses its own default, which is what it did before. + #[tokio::test] + async fn an_absent_snapshotter_is_not_an_error() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = runtime_file(&dir, r#"{"namespace":"dstack","project":"app"}"#); + let runtime = ComposeRuntime::read_from(&path) + .await + .expect("read") + .expect("present"); + assert_eq!(runtime.snapshotter, None); + } + + /// An app is free to move its containers to another namespace. Everything + /// it does is measured, so that is its choice -- but the agent has to look + /// where they actually are rather than where it assumed. + #[tokio::test] + async fn a_non_default_namespace_is_honoured() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = runtime_file(&dir, r#"{"namespace":"tenant-ns","project":"app"}"#); + let runtime = ComposeRuntime::read_from(&path) + .await + .expect("read") + .expect("present"); + assert_eq!(runtime.namespace, "tenant-ns"); + } + + /// `app-compose.sh` writes this before it starts anything, so its absence + /// is the boot window -- reported as "nothing created yet", not as an error + /// and not as a pass. + #[tokio::test] + async fn an_absent_runtime_file_is_the_boot_window() { + let missing = ComposeRuntime::read_from("/nonexistent/app-compose-runtime.json") + .await + .expect("absence is not an error"); + assert!(missing.is_none()); + let report = judge(vec![]); + assert!(!report.healthy); + } + + #[tokio::test] + async fn a_runtime_file_naming_no_project_is_an_error() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = runtime_file(&dir, r#"{"namespace":"dstack","project":""}"#); + assert!(ComposeRuntime::read_from(&path).await.is_err()); + } + + #[test] + fn the_filter_is_the_label_both_runtimes_set() { + assert_eq!( + compose_filter("myapp"), + "com.docker.compose.project=myapp", + "nerdctl sets only project/service/config-hash, so this is the only \ + project label the two runtimes agree on" + ); + } + + /// The window this whole feature exists for: the CVM has registered, the + /// runtime is answering, and Compose has not created anything yet. + #[test] + fn no_container_yet_is_not_healthy() { + let report = judge(vec![]); + assert!(!report.healthy); + assert_eq!(report.unhealthy.len(), 1); + assert_eq!(report.unhealthy[0].name, "<compose>"); + } + + /// An app that asked to be gated on an answer nothing here can produce. + /// + /// Passing it would be the silent no-op this feature exists to remove: the + /// gate is switched on, does nothing, and the only signal is the absence of + /// an effect. Naming `<compose>` puts a reason in the gateway log and on + /// the dashboard instead. This is judged here rather than by scanning the + /// compose file at deploy time because only the runtime knows -- a + /// `HEALTHCHECK` in a Dockerfile is nowhere in the compose file, and + /// `healthcheck: {disable: true}` is in it while producing nothing. + /// + /// The three shapes are the ones both runtimes actually emit for "no + /// healthcheck": docker says `none`, nerdctl omits `State.Health` + /// altogether, and an empty string is the defensive third. + #[test] + fn an_app_where_no_container_declares_a_healthcheck_is_reported_not_passed() { + let report = judge(vec![ + named("web", None), + named("db", Some("none")), + named("init", Some("")), + ]); + assert!(!report.healthy); + assert_eq!(report.unhealthy.len(), 1); + assert_eq!(report.unhealthy[0].name, "<compose>"); + assert!( + report.unhealthy[0].status.contains("nothing can be judged"), + "unexpected status: {}", + report.unhealthy[0].status + ); + } + + /// The carve-out the rule above must not swallow. A container declaring no + /// healthcheck is still skipped *individually* as long as something else in + /// the project is judged -- otherwise every project with a sidecar or an + /// init container would fail on the sidecar. + #[test] + fn a_container_without_a_healthcheck_is_skipped_when_another_is_judged() { + let report = judge(vec![ + named("web", Some("healthy")), + named("sidecar", None), + named("db", Some("none")), + ]); + assert!(report.healthy); + assert!(report.unhealthy.is_empty(), "{:?}", report.unhealthy); + } + + #[test] + fn every_declared_healthcheck_passing_is_healthy() { + let report = judge(vec![named("web", Some("healthy")), named("sidecar", None)]); + assert!(report.healthy); + } + + /// `starting` is the boot window, so it must not count as ready. + #[test] + fn a_starting_container_is_not_healthy() { + let report = judge(vec![ + named("web", Some("healthy")), + named("api", Some("starting")), + ]); + assert!(!report.healthy); + assert_eq!(report.unhealthy[0].name, "api"); + assert_eq!(report.unhealthy[0].status, "starting"); + } + + /// The case the run-state check exists for. Captured behaviour, not a + /// hypothesis: on nerdctl 2.3.5 a container that passed its check and then + /// exited still reports `healthy`, so judging on the health string alone + /// keeps a dead app in the rotation. + #[test] + fn a_container_that_passed_its_check_and_then_exited_is_not_healthy() { + let report = judge(vec![stopped("web", "exited", Some("healthy"))]); + assert!(!report.healthy); + assert_eq!(report.unhealthy[0].name, "web"); + assert_eq!( + report.unhealthy[0].status, + "exited (last health check: healthy)" + ); + } + + /// Not running covers more than exited: a paused or endlessly restarting + /// container is not serving either, and neither runtime is relied on to + /// have rewritten the verdict. + #[test] + fn a_paused_or_restarting_container_is_not_healthy() { + for state in ["paused", "restarting", "created", "dead"] { + let report = judge(vec![stopped("web", state, Some("healthy"))]); + assert!(!report.healthy, "{state} should not count as healthy"); + } + } + + /// The carve-out that must survive: an app that declares no healthcheck for + /// a one-shot init container is not failed when that container exits, which + /// is what it was written to do. + #[test] + fn a_container_without_a_healthcheck_may_exit() { + let report = judge(vec![ + named("web", Some("healthy")), + stopped("init", "exited", None), + ]); + assert!(report.healthy); + } + + /// A runtime that stops reporting `State.Status` must not take every + /// instance out of rotation; the health string still decides. + #[test] + fn a_missing_run_state_falls_back_to_the_health_status() { + let unknown = |name: &str, status: Option<&str>| Observed { + name: name.to_string(), + health: status.map(str::to_string), + state: None, + }; + assert!(judge(vec![unknown("web", Some("healthy"))]).healthy); + assert!(!judge(vec![unknown("web", Some("unhealthy"))]).healthy); + } + + #[test] + fn one_unhealthy_container_fails_the_instance() { + let report = judge(vec![ + named("web", Some("healthy")), + named("api", Some("unhealthy")), + ]); + assert!(!report.healthy); + assert_eq!(report.unhealthy[0].status, "unhealthy"); + } + + /// Fail closed: a state neither runtime documents today must hold traffic + /// back rather than be waved through as "not healthy but not a problem". + #[test] + fn an_unrecognised_state_holds_traffic_back() { + let report = judge(vec![named("web", Some("degraded"))]); + assert!(!report.healthy); + assert_eq!(report.unhealthy[0].status, "degraded"); + } + + #[test] + fn docker_names_lose_their_leading_slash() { + assert_eq!( + container_name(Some(&["/app-web-1".to_string()]), "id"), + "app-web-1" + ); + assert_eq!(container_name(None, "fallback-id"), "fallback-id"); + } + + /// Captured from a real `nerdctl --namespace ... inspect` on nerdctl 2.3.5, + /// trimmed of the fields this does not read. + /// + /// Two details this pins, both confirmed on hardware rather than assumed: + /// `Name` has **no** leading slash (Docker's does), and `State.Health` is + /// absent entirely -- not `"none"` -- for a container that declares no + /// healthcheck. + const NERDCTL_INSPECT_2_3_5: &str = r#"[ + { + "Id": "aed39a5956f7cf11bb438296e804c712d038678a05627ff97f70d5f691eafe17", + "Created": "2026-08-20T04:50:32.726557612Z", + "Name": "nchp-failcheck-1", + "Image": "docker.io/library/busybox:latest", + "Platform": "linux", + "State": { + "Status": "running", "Running": true, "Paused": false, + "Restarting": false, "Pid": 5157, "ExitCode": 0, "Error": "", + "StartedAt": "2026-08-20T04:50:32.817445652Z", "FinishedAt": "", + "Health": { + "Status": "unhealthy", "FailingStreak": 17, + "Log": [{"Start": "2026-08-20T04:51:17.56949457Z", + "End": "2026-08-20T04:51:17.602166707Z", + "ExitCode": 1, "Output": ""}] + } + } + }, + { + "Id": "8b92c5ef3f1855d5c35f37b8e7dd3ffe6b928e919a8b749409f707bf0addee05", + "Created": "2026-08-20T04:50:32.52880536Z", + "Name": "nchp-nocheck-1", + "Image": "docker.io/library/busybox:latest", + "Platform": "linux", + "State": { + "Status": "running", "Running": true, "Paused": false, + "Restarting": false, "Pid": 5063, "ExitCode": 0, "Error": "", + "StartedAt": "2026-08-20T04:50:32.627815194Z", "FinishedAt": "" + } + }, + { + "Id": "6e7d785b3b5b05fe55f2b0ec07762aa387435b359bda185f925d44f00381f3b5", + "Created": "2026-08-20T04:50:32.294111652Z", + "Name": "nchp-withcheck-1", + "Image": "docker.io/library/busybox:latest", + "Platform": "linux", + "State": { + "Status": "running", "Running": true, "Paused": false, + "Restarting": false, "Pid": 4921, "ExitCode": 0, "Error": "", + "StartedAt": "2026-08-20T04:50:32.401211753Z", "FinishedAt": "", + "Health": { + "Status": "healthy", "FailingStreak": 0, + "Log": [{"Start": "2026-08-20T04:51:18.12Z", + "End": "2026-08-20T04:51:18.15Z", + "ExitCode": 0, "Output": ""}] + } + } + } + ]"#; + + /// Parse the real payload the way `collect_nerdctl` does. + fn parse_nerdctl(raw: &str) -> Vec<Observed> { + let containers: Vec<NerdctlContainer> = serde_json::from_str(raw).unwrap(); + containers.into_iter().map(observe_nerdctl).collect() + } + + #[test] + fn real_nerdctl_inspect_output_parses() { + let observed = parse_nerdctl(NERDCTL_INSPECT_2_3_5); + assert_eq!( + observed, + vec![ + named("nchp-failcheck-1", Some("unhealthy")), + // No healthcheck declared: no Health object at all. + named("nchp-nocheck-1", None), + named("nchp-withcheck-1", Some("healthy")), + ] + ); + } + + /// Captured from the same nerdctl 2.3.5 after `nerdctl stop`, and the + /// reason [`is_running`] exists: the container is `exited` with + /// `"Running": false`, and `State.Health.Status` is still `healthy` -- + /// nerdctl never rewrites the stored verdict, because + /// `container healthcheck` returns "container is not running" before it + /// gets that far. A container that crashes on its own produces the same + /// pair, which is the shape this actually has to catch in production. + const NERDCTL_INSPECT_2_3_5_STOPPED: &str = r#"[ + { + "Id": "28e06fc13e2a6c487949b3291e319b7a8e88e155b00eb8cce33c620d95e87d03", + "Created": "2026-08-20T15:15:36.647689318Z", + "Name": "hc", + "Image": "docker.io/library/busybox:latest", + "Platform": "linux", + "State": { + "Status": "exited", "Running": false, "Paused": false, + "Restarting": false, "Pid": 0, "ExitCode": 137, "Error": "", + "StartedAt": "2026-08-20T15:15:36.968241457Z", + "FinishedAt": "2026-08-20T15:16:08.374095926Z", + "Health": { + "Status": "healthy", "FailingStreak": 0, + "Log": [{"Start": "2026-08-20T08:15:37.102770955-07:00", + "End": "2026-08-20T08:15:37.162666079-07:00", + "ExitCode": 0, "Output": ""}] + } + } + } + ]"#; + + /// Over the captured payload rather than a hand-built one: the guest agent + /// must not report this instance healthy. + #[test] + fn a_real_stopped_nerdctl_container_keeps_its_healthy_verdict_but_is_judged_by_run_state() { + let observed = parse_nerdctl(NERDCTL_INSPECT_2_3_5_STOPPED); + assert_eq!( + observed, + vec![stopped("hc", "exited", Some("healthy"))], + "nerdctl still reports the stale verdict; that is the point" + ); + + let report = judge(observed); + assert!(!report.healthy); + assert_eq!(report.unhealthy[0].name, "hc"); + assert_eq!( + report.unhealthy[0].status, + "exited (last health check: healthy)" + ); + } + + /// End to end over the real payload: one failing healthcheck holds the + /// instance out, and the container without one does not save it. + #[test] + fn a_real_nerdctl_project_with_a_failing_check_is_not_healthy() { + let report = judge(parse_nerdctl(NERDCTL_INSPECT_2_3_5)); + assert!(!report.healthy); + assert_eq!(report.unhealthy.len(), 1); + assert_eq!(report.unhealthy[0].name, "nchp-failcheck-1"); + assert_eq!(report.unhealthy[0].status, "unhealthy"); + } +} diff --git a/dstack/guest-agent/src/gpu_attest.rs b/dstack/guest-agent/src/gpu_attest.rs new file mode 100644 index 000000000..9d8013a25 --- /dev/null +++ b/dstack/guest-agent/src/gpu_attest.rs @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! On-demand GPU attestation for the `AttestGpu` RPC. +//! +//! Running `nvattest` is expensive in a way most RPCs are not: it spawns a +//! process and talks to the GPU through the driver. The gate below serialises +//! collection so concurrent callers do not compete for the same devices. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{bail, Result}; +use tokio::sync::Mutex; + +/// Serialises GPU evidence collection. +/// +/// Results are deliberately not shared between waiters: each caller supplies +/// its own nonce, and handing back evidence that answers somebody else's +/// challenge is exactly the confusion this API exists to avoid. +pub struct GpuAttestor { + timeout: Duration, + /// The binary whose presence says this image can attest a GPU. + /// + /// A field rather than a call to `nvattest::available()` so a test can pin + /// the unavailable answer on a host that happens to have nvattest + /// installed -- otherwise that test would spawn a real collection against + /// whatever GPUs the host has. + nvattest: PathBuf, + run_lock: Mutex<()>, +} + +impl GpuAttestor { + pub fn new() -> Self { + Self::with_nvattest_path(nvattest::NVATTEST) + } + + /// `pub(crate)` so the handler tests can build a state that is + /// deterministically without GPU support, whatever the host running + /// them has installed. + pub(crate) fn with_nvattest_path(path: impl AsRef<Path>) -> Self { + Self { + timeout: nvattest::DEFAULT_TIMEOUT, + nvattest: path.as_ref().to_path_buf(), + run_lock: Mutex::new(()), + } + } + + /// Attest against `nonce`, or explain why not. + pub async fn attest(&self, nonce: &[u8]) -> Result<Vec<u8>> { + if nonce.len() != nvattest::NONCE_LEN { + bail!( + "nonce must be exactly {} bytes, got {}", + nvattest::NONCE_LEN, + nonce.len() + ); + } + if !self.nvattest.exists() { + // 501, not the 400 an uncoded error would report. The request is + // well-formed and no other request would succeed: this image ships + // no nvattest, so the capability is absent for the lifetime of the + // CVM. A caller that reads 400 retries with different arguments + // forever; one that reads 501 stops and falls back. + // + // Safe to say now because `AttestGpu` is v1-only and has never + // shipped in a release -- no client is reading 400 here today. + ra_rpc::bail!(501, "GPU attestation is not available in this image"); + } + // Held across the whole run, so a second caller waits rather than + // starting a competing nvattest against the same devices. + let _guard = self.run_lock.lock().await; + let (_, evidence) = nvattest::collect_evidence(nonce, self.timeout).await?; + Ok(evidence) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn a_wrong_length_nonce_is_rejected_before_anything_expensive() { + let attestor = GpuAttestor::new(); + let err = attestor.attest(&[0u8; 16]).await.unwrap_err().to_string(); + assert!(err.contains("exactly 32 bytes"), "{err}"); + } + + /// A malformed nonce is the caller's fault; an image without nvattest is + /// not. The two must not answer with the same status. + /// + /// The nonce here is well-formed, so this reaches the availability check + /// rather than stopping at the length check above. + #[tokio::test] + async fn an_image_without_nvattest_answers_not_implemented() { + let attestor = GpuAttestor::with_nvattest_path("/nonexistent/nvattest"); + let err = attestor.attest(&[0u8; 32]).await.unwrap_err(); + assert!( + err.to_string().contains("not available in this image"), + "{err}" + ); + assert_eq!(ra_rpc::code_of(&err), Some(501)); + + let malformed = attestor.attest(&[0u8; 16]).await.unwrap_err(); + assert_eq!(ra_rpc::code_of(&malformed), None, "a bad nonce stays a 400"); + } +} diff --git a/dstack/guest-agent/src/gpu_info.rs b/dstack/guest-agent/src/gpu_info.rs new file mode 100644 index 000000000..6e62a9641 --- /dev/null +++ b/dstack/guest-agent/src/gpu_info.rs @@ -0,0 +1,478 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! GPU telemetry, sampled out of process and cached. +//! +//! Three properties drive the shape of this module: +//! +//! 1. **A guest without an NVIDIA card pays nothing.** PCI topology is fixed +//! for a CVM's lifetime -- the VMM assigns GPUs through VFIO when QEMU +//! starts and never hot-plugs -- so the scan runs once and every later +//! request is an atomic load returning a constant. +//! 2. **NVML never runs in this process.** `dstack-util gpu-info` samples in a +//! short-lived child that can be killed when a driver call wedges. Nothing +//! stays resident between samples, and each sample re-initializes NVML, so a +//! driver that loads late is picked up instead of being cached as "no GPU". +//! 3. **A request never waits on the GPU.** Callers get the last known snapshot +//! with its age attached and a refresh is kicked off behind them. Returning +//! a placeholder on expiry would mean a Prometheus scrape slower than the +//! TTL -- which is every realistic scrape interval -- never sees a single +//! GPU series. +//! +//! The cache is not here because sampling is slow. One `dstack-util gpu-info` +//! run, fork to exit, measured 94-99 ms on a single H200 in CC mode. It is here +//! because that number describes the healthy path only, and three things do not +//! follow from it: +//! +//! - `/metrics` also carries CPU, memory, disk and container state. Sampling +//! inline would put all of it behind a driver call that can wedge for +//! [`SAMPLE_TIMEOUT`], well past a default Prometheus scrape timeout, so one +//! stuck card would erase every metric this CVM reports rather than just the +//! GPU ones. +//! - `/metrics`, the dashboard and the `GpuInfo` RPC can arrive together, and +//! each would otherwise fork its own collector to repeat the same `dlopen` +//! and `nvmlInit_v2`. [`REFRESH_LOCK`] collapses a burst into one sample. +//! - Enumeration cost grows with card count, and only one card has been +//! measured. +//! +//! Refreshing is lazy: nothing resamples unless someone asks. A served snapshot +//! is therefore roughly one scrape interval old, not [`SAMPLE_TTL`] old -- the +//! TTL decides when a request triggers the next sample, not how fresh the +//! answer is. `sample_age_ms`, and `dstack_gpu_sample_age_seconds` on +//! `/metrics`, carry that age so a consumer can correct for it. A background +//! ticker would cap the age at the TTL instead, at the price of a permanent +//! ~2% of a core on every GPU guest whether or not anyone is watching, to +//! publish a number the consumer can already derive. + +use std::path::Path; +use std::process::Stdio; +use std::sync::{LazyLock, OnceLock}; +use std::time::Duration; + +use anyhow::{bail, Context, Result}; +use cached_cell::TtlCell; +use guest_api::GpuInfoResponse; +#[cfg(test)] +use or_panic::ResultOrPanic; +use tokio::process::Command; +use tokio::sync::Mutex; +use tokio::time::timeout; +use tracing::{debug, warn}; + +/// How long a successful snapshot is served before a request triggers a +/// refresh. Older snapshots are still served, just with a refresh started +/// behind them, so this bounds when resampling starts and not how old a served +/// sample can be. +const SAMPLE_TTL: Duration = Duration::from_secs(5); +/// Backoff after a failed sample. +/// +/// A collector that hangs burns [`SAMPLE_TIMEOUT`] and is then killed. Retrying +/// that on the success cadence would keep a guest whose driver has wedged in a +/// near-continuous spawn-and-kill loop, which is both useless and the state +/// most likely to leave a process stuck in an uninterruptible driver call. +/// Long enough to stop hammering, short enough that a recovered driver is +/// picked up while an operator is still looking at the dashboard. +const FAILURE_BACKOFF: Duration = Duration::from_secs(30); +/// Upper bound on one `dstack-util gpu-info` run. Finite because the whole +/// point of the child process is that a wedged driver can be abandoned. +/// +/// A measured run on a single H200 in CC mode takes 94-99 ms, so this is two +/// orders of magnitude of headroom. The margin stays until a multi-GPU CVM has +/// been measured: `nvmlInit_v2` enumerates every card, and trading the untested +/// case for faster recovery in the tested one is the wrong direction. +const SAMPLE_TIMEOUT: Duration = Duration::from_secs(10); + +/// The collector, installed into the rootfs alongside this agent. +const DSTACK_UTIL: &str = "/usr/bin/dstack-util"; +/// Set by tests to point at a stub collector. +#[cfg(test)] +static COLLECTOR_OVERRIDE: std::sync::RwLock<Option<String>> = std::sync::RwLock::new(None); + +/// Holds the last outcome, successful or not. The cell's own TTL is unused: +/// serving is always allowed and [`refresh_interval`] decides when to resample. +static SNAPSHOT: LazyLock<TtlCell<GpuInfoResponse>> = LazyLock::new(|| TtlCell::new(SAMPLE_TTL)); +/// Serializes refreshes so a burst of scrapes spawns one collector, not N. +static REFRESH_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(())); + +/// Why this guest cannot produce GPU telemetry, if it cannot. +enum Gate { + /// No NVIDIA device on the PCI bus. Nothing to report, ever. + NoGpu, + /// A card is present but the kernel module is not loaded yet. + NoDriver, + /// Sampling is worth attempting. + Sample, +} + +fn gate() -> Gate { + if !nvidia_on_pci() { + return Gate::NoGpu; + } + // Unlike PCI presence this is not fixed: the module can load after the + // agent starts, so it is re-checked every time. It is one `stat`. + if !Path::new("/sys/module/nvidia").exists() { + return Gate::NoDriver; + } + Gate::Sample +} + +/// True when an NVIDIA display device is attached to the PCI bus. +/// +/// Cached for the process lifetime. A CVM's GPUs are fixed at launch by the +/// VMM's VFIO assignment; there is no hot-plug path that could change this +/// answer, and paying a directory scan on every scrape of every GPU-less guest +/// is exactly the cost this gate exists to avoid. +/// +/// Fails open to "no GPU" where the boot attestation gate in `dstack-util` +/// fails closed on the same inventory: refusing to report telemetry is the +/// safe direction here, and spawning a collector forever on a guest whose +/// sysfs cannot be read is not. +fn nvidia_on_pci() -> bool { + static PRESENT: OnceLock<bool> = OnceLock::new(); + *PRESENT.get_or_init(|| match lspci::sysfs::gpu_inventory() { + Ok(inventory) => inventory.has_nvidia(), + Err(error) => { + warn!("failed to scan PCI for GPUs, assuming none: {error:#}"); + false + } + }) +} + +/// Returns the best answer available without ever blocking. +/// +/// Serves the last snapshot whatever its age, annotated with `sample_age_ms`, +/// and starts a refresh behind the caller when that snapshot has aged past the +/// TTL. Used by `/metrics` and the dashboard, where a wedged GPU must not +/// delay unrelated guest data. Only the very first call on a GPU guest returns +/// "not sampled yet". +pub(crate) fn gpu_info() -> GpuInfoResponse { + match gate() { + Gate::NoGpu => return no_gpus(), + Gate::NoDriver => return unavailable("NVIDIA driver is not loaded"), + Gate::Sample => {} + } + + let cached = SNAPSHOT.get_allow_stale().ok(); + if is_due(cached.as_ref()) { + tokio::spawn(refresh_if_free()); + } + serve(cached) +} + +/// Like [`gpu_info`], but waits for a first sample when the cache is cold. +/// +/// The `GpuInfo` RPC is a direct question from an operator or the control +/// plane, so "ask again in five seconds" is a worse answer than a bounded +/// wait. The wait is bounded twice: by [`SAMPLE_TIMEOUT`] here and by the RPC +/// timeout at the call site. +pub(crate) async fn gpu_info_awaited() -> GpuInfoResponse { + match gate() { + Gate::NoGpu => return no_gpus(), + Gate::NoDriver => return unavailable("NVIDIA driver is not loaded"), + Gate::Sample => {} + } + + if SNAPSHOT.get_allow_stale().is_err() { + // Cold cache. Queue behind any in-flight sample rather than reporting + // nothing; whoever wins the lock fills the cache for both. + let _guard = REFRESH_LOCK.lock().await; + if SNAPSHOT.get_allow_stale().is_err() { + sample_into_cache().await; + } + } + gpu_info() +} + +/// How long this outcome is kept before resampling. +fn refresh_interval(response: &GpuInfoResponse) -> Duration { + if response.error.is_empty() { + SAMPLE_TTL + } else { + FAILURE_BACKOFF + } +} + +/// True when the cached outcome has outlived its refresh interval. +fn is_due(cached: Option<&cached_cell::Snapshot<GpuInfoResponse>>) -> bool { + cached.is_none_or(|snapshot| snapshot.age() >= refresh_interval(snapshot.value())) +} + +fn serve(cached: Option<cached_cell::Snapshot<GpuInfoResponse>>) -> GpuInfoResponse { + match cached { + Some(snapshot) => { + let mut response = snapshot.value().clone(); + response.sample_age_ms = Some(snapshot.age().as_millis() as u64); + response + } + None => unavailable("GPU sample is not available yet"), + } +} + +/// Runs one collection unless another one is already in flight. +/// +/// Dropping the refresh when the lock is held is deliberate: a burst of scrapes +/// must spawn one collector, not one per scrape, and the waiting callers are +/// already being served the previous snapshot. +async fn refresh_if_free() { + let Ok(_guard) = REFRESH_LOCK.try_lock() else { + debug!("GPU sample already in progress, skipping refresh"); + return; + }; + // Another task may have refreshed between the staleness check and here. + if !is_due(SNAPSHOT.get_allow_stale().ok().as_ref()) { + return; + } + sample_into_cache().await; +} + +/// Collects once and stores the outcome, success or failure. +/// +/// Failures are cached like successes so a guest whose driver is broken reports +/// the reason instead of an empty device list, and so a hard-failing collector +/// is not re-spawned on every single scrape. +/// +/// Caller must hold [`REFRESH_LOCK`]. +async fn sample_into_cache() { + match timeout(SAMPLE_TIMEOUT, collect()).await { + Ok(Ok(response)) => { + SNAPSHOT.set(response); + } + Ok(Err(error)) => { + warn!("failed to sample GPU telemetry: {error:#}"); + SNAPSHOT.set(unavailable(format!("GPU sampling failed: {error:#}"))); + } + Err(_) => { + warn!("GPU sampling timed out after {SAMPLE_TIMEOUT:?}"); + SNAPSHOT.set(unavailable("GPU sampling timed out")); + } + } +} + +/// Spawns `dstack-util gpu-info` and parses its stdout. +/// +/// `kill_on_drop` matters: the timeout above drops this future, and a wedged +/// NVML call inside the child must not outlive it. +async fn collect() -> Result<GpuInfoResponse> { + let collector = collector_path(); + if !Path::new(&collector).exists() { + bail!("{collector} is not installed"); + } + let output = Command::new(&collector) + .arg("gpu-info") + .stdin(Stdio::null()) + .kill_on_drop(true) + .output() + .await + .with_context(|| format!("failed to run {collector} gpu-info"))?; + // The collector logs to stderr and reserves stdout for the document, so + // anything here is diagnostics worth keeping rather than protocol noise. + if !output.stderr.is_empty() { + warn!( + "gpu-info collector: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + if !output.status.success() { + bail!("gpu-info collector exited with {}", output.status); + } + serde_json::from_slice(&output.stdout).context("invalid gpu-info output") +} + +fn collector_path() -> String { + #[cfg(test)] + if let Some(path) = COLLECTOR_OVERRIDE + .read() + .or_panic("collector override poisoned") + .clone() + { + return path; + } + DSTACK_UTIL.to_string() +} + +/// The guest has no NVIDIA hardware. Empty devices with no error is the +/// documented encoding for "the collector ran and found nothing". +fn no_gpus() -> GpuInfoResponse { + GpuInfoResponse::default() +} + +fn unavailable(error: impl Into<String>) -> GpuInfoResponse { + GpuInfoResponse { + error: error.into(), + ..Default::default() + } +} + +/// Test-only hooks. Production callers go through [`gpu_info`]. +#[cfg(test)] +mod test_support { + use super::*; + + /// Serializes the tests that reach into process-global state. + /// + /// [`SNAPSHOT`], [`REFRESH_LOCK`] and [`COLLECTOR_OVERRIDE`] are one per + /// process while the harness runs tests on parallel threads, so without + /// this two tests race over the same cache entry and the same collector + /// path -- one pointing at a stub that sleeps, the other at a path that + /// does not exist. + static EXCLUSION: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(())); + + /// Held for the whole test body, not just the mutation, and restores the + /// collector override on the way out so a stub cannot leak into a test + /// that expects the real path. + pub(super) struct Exclusive { + _exclusion: tokio::sync::MutexGuard<'static, ()>, + _refresh: tokio::sync::MutexGuard<'static, ()>, + } + + impl Drop for Exclusive { + fn drop(&mut self) { + *COLLECTOR_OVERRIDE + .write() + .or_panic("collector override poisoned") = None; + } + } + + pub(super) async fn exclusive() -> Exclusive { + let exclusion = EXCLUSION.lock().await; + // Waits rather than try_lock: a refresh spawned by an earlier test may + // still be finishing, and failing a test for losing that race tests + // the harness rather than the code. + let refresh = REFRESH_LOCK.lock().await; + Exclusive { + _exclusion: exclusion, + _refresh: refresh, + } + } + + pub(super) fn set_snapshot(response: GpuInfoResponse) { + SNAPSHOT.set(response); + } + + pub(super) fn set_collector(path: &str) { + *COLLECTOR_OVERRIDE + .write() + .or_panic("collector override poisoned") = Some(path.to_string()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use guest_api::GpuDevice; + use std::os::unix::fs::PermissionsExt; + + fn sample_response() -> GpuInfoResponse { + GpuInfoResponse { + gpus: vec![GpuDevice { + index: 0, + uuid: "GPU-abc".into(), + ..Default::default() + }], + ..Default::default() + } + } + + /// A guest with no NVIDIA device must not scan, spawn, or cache anything. + /// The distinction that matters downstream is "no GPUs" (empty, no error) + /// versus "could not tell" (error set). + #[test] + fn a_guest_without_a_card_reports_no_gpus_rather_than_an_error() { + let response = no_gpus(); + assert!(response.gpus.is_empty()); + assert!(response.error.is_empty()); + assert_eq!(response.sample_age_ms, None); + } + + /// A failed sample must not be retried on the success cadence. A collector + /// that burns the whole timeout and gets killed would otherwise be respawned + /// on every scrape, which is the state most likely to strand a process in an + /// uninterruptible driver call. + #[test] + fn a_failed_sample_backs_off_further_than_a_successful_one() { + assert_eq!(refresh_interval(&sample_response()), SAMPLE_TTL); + assert_eq!( + refresh_interval(&unavailable("GPU sampling timed out")), + FAILURE_BACKOFF + ); + assert!(FAILURE_BACKOFF > SAMPLE_TTL); + } + + /// An empty cache is always due; that is the cold-start path. + #[test] + fn an_empty_cache_is_due() { + assert!(is_due(None)); + } + + /// The bug this design exists to prevent: a scrape interval longer than the + /// TTL must still see the GPU series. Serving a placeholder on expiry meant + /// `/metrics` reported zero GPUs forever at Prometheus' default 15s. + #[tokio::test] + async fn a_snapshot_older_than_the_ttl_is_still_served() { + // Hold the globals so no refresh, and no other test, can overwrite the + // snapshot mid-assertion on a machine that does have a GPU. + let _guard = test_support::exclusive().await; + test_support::set_snapshot(sample_response()); + + let served = serve(SNAPSHOT.get_allow_stale().ok()); + + assert!( + served.error.is_empty(), + "stale data must not become an error" + ); + assert_eq!(served.gpus.len(), 1); + assert!(served.sample_age_ms.is_some(), "age must be reported"); + } + + /// The reason sampling lives in a child process: when a driver call wedges, + /// the timeout must actually reclaim it. Nothing may be left running + /// between samples. + #[tokio::test] + async fn a_collector_that_hangs_is_killed_when_the_sample_times_out() { + let dir = tempfile::tempdir().expect("tempdir"); + let script = dir.path().join("stub-collector"); + let pid_file = dir.path().join("pid"); + std::fs::write( + &script, + format!("#!/bin/sh\necho $$ > {}\nsleep 60\n", pid_file.display()), + ) + .expect("write stub"); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)) + .expect("chmod stub"); + + let _guard = test_support::exclusive().await; + test_support::set_collector(&script.to_string_lossy()); + + let outcome = timeout(Duration::from_millis(500), collect()).await; + assert!(outcome.is_err(), "the stub sleeps far past the timeout"); + + let pid: i32 = std::fs::read_to_string(&pid_file) + .expect("stub recorded its pid") + .trim() + .parse() + .expect("pid is a number"); + + // SIGKILL is asynchronous; give the kernel a moment to reap. + for _ in 0..50 { + if !Path::new(&format!("/proc/{pid}")).exists() { + return; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + panic!("collector {pid} survived the sample timeout"); + } + + /// A collector that is not installed must degrade to a recorded error, not + /// a panic and not a silent "no GPUs". + #[tokio::test] + async fn a_missing_collector_is_reported_as_an_error() { + let _guard = test_support::exclusive().await; + test_support::set_collector("/nonexistent/dstack-util"); + let error = collect().await.expect_err("must fail"); + assert!( + error.to_string().contains("not installed"), + "unexpected error: {error:#}" + ); + } +} diff --git a/dstack/guest-agent/src/guest_api_service.rs b/dstack/guest-agent/src/guest_api_service.rs index 8306a104b..cf3e96163 100644 --- a/dstack/guest-agent/src/guest_api_service.rs +++ b/dstack/guest-agent/src/guest_api_service.rs @@ -2,26 +2,33 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::fmt::Debug; +use std::{collections::HashSet, fmt::Debug, path::PathBuf, process::ExitStatus, time::Duration}; use anyhow::{Context, Result}; use bollard::{container::ListContainersOptions, Docker}; -use cmd_lib::{run_cmd as cmd, run_fun}; -use dstack_guest_agent_rpc::worker_server::WorkerRpc as _; +use dstack_guest_agent_rpc::v0::worker_server::WorkerRpc as _; use dstack_types::shared_filenames::{HOST_SHARED_DIR, SYS_CONFIG}; use dstack_types::SysConfig; use fs_err as fs; use guest_api::{ guest_api_server::{GuestApiRpc, GuestApiServer}, - Container, DiskInfo, Gateway, GuestInfo, Interface, IpAddress, ListContainersResponse, - NetworkInformation, SystemInfo, + Container, DiskInfo, Gateway, GpuInfoResponse, GuestInfo, Interface, IpAddress, + ListContainersResponse, NetworkInformation, SystemInfo, }; use host_api::Notification; use ra_rpc::{CallContext, RpcCall}; +use tokio::{process::Command, task::spawn_blocking, time::timeout}; use tracing::error; use crate::{rpc_service::ExternalRpcHandler, AppState}; +const GUEST_API_HANDLER_TIMEOUT: Duration = Duration::from_secs(20); +const DOCKER_API_TIMEOUT: Duration = Duration::from_secs(15); +const SHUTDOWN_NOTIFY_TIMEOUT: Duration = Duration::from_secs(5); +const POWEROFF_COMMAND_TIMEOUT: Duration = Duration::from_secs(10); +const WG_COMMAND_TIMEOUT: Duration = Duration::from_secs(5); +const GPU_INFO_TIMEOUT: Duration = Duration::from_secs(15); + pub struct GuestApiHandler { state: AppState, } @@ -39,7 +46,9 @@ impl RpcCall<AppState> for GuestApiHandler { impl GuestApiRpc for GuestApiHandler { async fn info(self) -> Result<GuestInfo> { let ext_rpc = ExternalRpcHandler::new(self.state); - let info = ext_rpc.info().await?; + let info = timeout(GUEST_API_HANDLER_TIMEOUT, ext_rpc.info()) + .await + .context("Guest Info request timed out")??; Ok(GuestInfo { version: env!("CARGO_PKG_VERSION").to_string(), app_id: info.app_id, @@ -52,79 +61,124 @@ impl GuestApiRpc for GuestApiHandler { async fn shutdown(self) -> Result<()> { tokio::spawn(async move { - notify_host("shutdown.progress", "powering off").await.ok(); - perr(cmd!(systemctl poweroff)); + let _ = timeout( + SHUTDOWN_NOTIFY_TIMEOUT, + notify_host("shutdown.progress", "powering off"), + ) + .await; + let mut command = Command::new("systemctl"); + command.arg("poweroff").kill_on_drop(true); + perr( + timeout(POWEROFF_COMMAND_TIMEOUT, command.status()) + .await + .context("systemctl poweroff timed out") + .and_then(|result| result.context("failed to run systemctl poweroff")) + .and_then(|status| ensure_command_success("systemctl poweroff", status)), + ); }); Ok(()) } async fn network_info(self) -> Result<NetworkInformation> { - Ok(NetworkInformation { - dns_servers: get_dns_servers(), - gateways: get_gateways(), - interfaces: get_interfaces(), - wg_info: get_wg_info().unwrap_or_else(|e| e.to_string()), - }) + let mut info = timeout( + GUEST_API_HANDLER_TIMEOUT, + spawn_blocking(collect_network_info), + ) + .await + .context("NetworkInfo request timed out")? + .context("NetworkInfo worker failed")??; + info.wg_info = get_wg_info().await.unwrap_or_else(|e| e.to_string()); + Ok(info) } async fn sys_info(self) -> Result<SystemInfo> { - use sysinfo::System; - - let system = System::new_all(); - let cpus = system.cpus(); - - let disks = sysinfo::Disks::new_with_refreshed_list(); - let config = self.state.config(); - let mut disks = disks - .list() - .iter() - .filter(|d| config.data_disks.contains(d.mount_point())) - .map(|d| DiskInfo { - name: d.name().to_string_lossy().to_string(), - mount_point: d.mount_point().to_string_lossy().to_string(), - total_size: d.total_space(), - free_size: d.available_space(), - }) - .collect::<Vec<_>>(); - disks.sort_by(|d1, d2| d1.mount_point.cmp(&d2.mount_point)); - let avg = System::load_average(); - Ok(SystemInfo { - os_name: System::name().unwrap_or_default(), - os_version: System::os_version().unwrap_or_default(), - kernel_version: System::kernel_version().unwrap_or_default(), - cpu_model: cpus.first().map_or("".into(), |cpu| { - format!("{} @{} MHz", cpu.name(), cpu.frequency()) - }), - num_cpus: cpus.len() as _, - total_memory: system.total_memory(), - available_memory: system.available_memory(), - used_memory: system.used_memory(), - free_memory: system.free_memory(), - total_swap: system.total_swap(), - used_swap: system.used_swap(), - free_swap: system.free_swap(), - uptime: System::uptime(), - loadavg_one: (avg.one * 100.0) as u32, - loadavg_five: (avg.five * 100.0) as u32, - loadavg_fifteen: (avg.fifteen * 100.0) as u32, - disks, - }) + let data_disks = self.state.config().data_disks.clone(); + timeout( + GUEST_API_HANDLER_TIMEOUT, + spawn_blocking(move || collect_sys_info(&data_disks)), + ) + .await + .context("SysInfo request timed out")? + .context("SysInfo worker failed") + } + + async fn gpu_info(self) -> Result<GpuInfoResponse> { + timeout(GPU_INFO_TIMEOUT, crate::gpu_info::gpu_info_awaited()) + .await + .context("GpuInfo request timed out") } async fn list_containers(self) -> Result<ListContainersResponse> { - list_containers().await + timeout(GUEST_API_HANDLER_TIMEOUT, list_containers()) + .await + .context("ListContainers request timed out")? + } +} + +fn collect_network_info() -> Result<NetworkInformation> { + Ok(NetworkInformation { + dns_servers: get_dns_servers(), + gateways: get_gateways(), + interfaces: get_interfaces(), + wg_info: String::new(), + }) +} + +fn collect_sys_info(data_disks: &HashSet<PathBuf>) -> SystemInfo { + use sysinfo::{Disks, System}; + + let system = System::new_all(); + let cpus = system.cpus(); + + let disks = Disks::new_with_refreshed_list(); + let mut disks = disks + .list() + .iter() + .filter(|d| data_disks.contains(d.mount_point())) + .map(|d| DiskInfo { + name: d.name().to_string_lossy().to_string(), + mount_point: d.mount_point().to_string_lossy().to_string(), + total_size: d.total_space(), + free_size: d.available_space(), + }) + .collect::<Vec<_>>(); + disks.sort_by(|d1, d2| d1.mount_point.cmp(&d2.mount_point)); + let avg = System::load_average(); + SystemInfo { + os_name: System::name().unwrap_or_default(), + os_version: System::os_version().unwrap_or_default(), + kernel_version: System::kernel_version().unwrap_or_default(), + cpu_model: cpus.first().map_or("".into(), |cpu| { + format!("{} @{} MHz", cpu.name(), cpu.frequency()) + }), + num_cpus: cpus.len() as _, + total_memory: system.total_memory(), + available_memory: system.available_memory(), + used_memory: system.used_memory(), + free_memory: system.free_memory(), + total_swap: system.total_swap(), + used_swap: system.used_swap(), + free_swap: system.free_swap(), + uptime: System::uptime(), + loadavg_one: (avg.one * 100.0) as u32, + loadavg_five: (avg.five * 100.0) as u32, + loadavg_fifteen: (avg.fifteen * 100.0) as u32, + disks, } } pub(crate) async fn list_containers() -> Result<ListContainersResponse> { let docker = Docker::connect_with_defaults().context("Failed to connect to Docker")?; - let containers = docker - .list_containers::<&str>(Some(ListContainersOptions { + let containers = timeout( + DOCKER_API_TIMEOUT, + docker.list_containers::<&str>(Some(ListContainersOptions { all: true, ..Default::default() - })) - .await - .context("Failed to list containers")?; + })), + ) + .await + .context("Docker list-containers request timed out")? + .context("Failed to list containers")?; Ok(ListContainersResponse { containers: containers .into_iter() @@ -145,11 +199,11 @@ fn get_interfaces() -> Vec<Interface> { sysinfo::Networks::new_with_refreshed_list() .into_iter() .filter_map(|(interface_name, network)| { - if !(interface_name == "dstack-wg0" + if !(interface_name.starts_with("dstack-wg") || interface_name.starts_with("enp") || interface_name.starts_with("eth")) { - // We only get dstack-wg0, enp and eth interfaces. + // We only get dstack gateway, enp and eth interfaces. // Docker bridge is not included due to privacy concerns. return None; } @@ -200,9 +254,22 @@ fn get_dns_servers() -> Vec<String> { dns_servers } -fn get_wg_info() -> Result<String> { - let output = run_fun!(wg)?; - Ok(output) +async fn get_wg_info() -> Result<String> { + let mut command = Command::new("wg"); + command.kill_on_drop(true); + let output = timeout(WG_COMMAND_TIMEOUT, command.output()) + .await + .context("wg command timed out")? + .context("failed to run wg")?; + ensure_command_success("wg", output.status)?; + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +fn ensure_command_success(command: &str, status: ExitStatus) -> Result<()> { + if !status.success() { + anyhow::bail!("{command} exited with {status}"); + } + Ok(()) } pub async fn notify_host(event: &str, payload: &str) -> Result<()> { @@ -226,3 +293,23 @@ fn perr<T, E: Debug>(result: Result<T, E>) { error!("{e:?}"); } } + +#[cfg(test)] +mod tests { + use super::ensure_command_success; + use std::{os::unix::process::ExitStatusExt, process::ExitStatus}; + + #[test] + fn command_status_accepts_zero_exit_code() { + assert!(ensure_command_success("test", ExitStatus::from_raw(0)).is_ok()); + } + + #[test] + fn command_status_rejects_nonzero_exit_code() { + let error = ensure_command_success("test", ExitStatus::from_raw(7 << 8)) + .expect_err("nonzero exit status must fail"); + assert!(error + .to_string() + .contains("test exited with exit status: 7")); + } +} diff --git a/dstack/guest-agent/src/health.rs b/dstack/guest-agent/src/health.rs new file mode 100644 index 000000000..5a2c94b9d --- /dev/null +++ b/dstack/guest-agent/src/health.rs @@ -0,0 +1,660 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Application health, refreshed on the agent's own timer. +//! +//! A CVM registers from `dstack-util setup`, during boot, before +//! `app-compose.service` has started a single container. From that moment the +//! gateway considers it eligible for traffic, and the only thing standing +//! between it and a request is the WireGuard handshake age -- which says the +//! tunnel is up and nothing about whether anything is listening behind it. +//! `Worker.Health` is how the gateway asks the second question. +//! +//! Everything expensive happens here, on a timer, and `Worker.Health` only +//! reads [`HealthMonitor::report`]'s cached answer. Two reasons that split is +//! not optional: +//! +//! - A fleet of gateway nodes polls the same instance. Doing the work per +//! request would multiply one operator's cluster size into that many +//! container-runtime queries inside every CVM. +//! - The RPC is served on the external listener, which is publicly reachable +//! (see `docs/security/cvm-boundaries.md`). Any work it does per request is +//! work an anonymous caller can ask for at an arbitrary rate. +//! +//! Which question gets asked is the app's choice, declared in +//! `requirements.health_check`: a `health_status_file` it writes itself, or -- by +//! default -- the state of the containers the runtime knows about. + +use std::sync::{Arc, RwLock}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use dstack_types::HEALTH_FILE_MAX_AGE_SECS; +use or_panic::ResultOrPanic; +use tracing::{debug, info, warn}; + +use dstack_guest_agent_rpc::v1::ContainerHealth; + +use crate::container_health; + +/// How often the verdict is recomputed. +/// +/// The gateway polls on its own interval; total detection latency is the sum of +/// the two. Kept short enough that the sum stays inside the window an operator +/// would call "noticed it", and long enough that the container path is a +/// handful of runtime queries a minute rather than one per gateway request. +const REFRESH_INTERVAL: Duration = Duration::from_secs(5); + +/// How long one refresh may take before it is abandoned. +/// +/// A wedged container runtime must not stall the loop forever, and the child +/// processes started underneath are spawned with `kill_on_drop`, so abandoning +/// the future actually reclaims them. +const REFRESH_TIMEOUT: Duration = Duration::from_secs(10); + +/// How many consecutive refresh failures are tolerated before the verdict flips. +/// +/// A single failure is far more often a container being recreated underneath us +/// than an app going down, and this loop gets another attempt in +/// [`REFRESH_INTERVAL`]. Ejecting an instance from rotation on the first one +/// would make every redeploy a brief outage. +const MAX_CONSECUTIVE_FAILURES: u32 = 3; + +/// Largest `health_status_file` the agent will read. +/// +/// The file is two short lines. Anything beyond this is either a mistake or an +/// attempt to make the agent allocate on the app's behalf. +const MAX_HEALTH_FILE_BYTES: u64 = 4096; + +/// Longest app-authored text echoed back into a report. +/// +/// Only the `health_status_file` path reaches a report now -- the file's *contents* +/// deliberately do not, since whatever sits at that path is not what was +/// measured. A path is short; bounding it keeps app-controlled bytes out of the +/// gateway's logs in quantity, and [`sanitize`] handles the rest. +const MAX_REPORTED_BYTES: usize = 256; + +/// Reported in place of a container name, so an operator reading gateway logs +/// can tell an app-written verdict from a container that failed its healthcheck. +const HEALTH_FILE_NAME: &str = "<health_file>"; + +/// Reported before either source has been consulted for the first time. +const PENDING_NAME: &str = "<health>"; + +/// The latest verdict, in the shape `Worker.Health` answers with. +/// +/// Carries the generated `ContainerHealth` rather than a local twin of it. The +/// two were copied field by field on the way out, which is a conversion whose +/// only failure mode is silent. +#[derive(Clone, Debug, Default, PartialEq)] +pub(crate) struct Verdict { + pub healthy: bool, + pub unhealthy: Vec<ContainerHealth>, + /// Set when the agent could not reach the source of truth at all, as + /// opposed to reaching it and being told something is wrong. + pub error: String, +} + +impl Verdict { + pub(crate) fn healthy() -> Self { + Self { + healthy: true, + unhealthy: vec![], + error: String::new(), + } + } + + /// Unhealthy for a reason the agent can name, attributed to `name`. + pub(crate) fn unhealthy(name: &str, status: String) -> Self { + Self { + healthy: false, + unhealthy: vec![ContainerHealth { + name: name.to_string(), + status, + }], + error: String::new(), + } + } + + /// Healthy exactly when nothing was found wrong. + pub(crate) fn from_containers(unhealthy: Vec<ContainerHealth>) -> Self { + Self { + healthy: unhealthy.is_empty(), + unhealthy, + error: String::new(), + } + } + + /// Unhealthy because the agent could not see the app at all. + fn failed(error: String) -> Self { + Self { + healthy: false, + unhealthy: vec![], + error, + } + } + + /// One line, for the log emitted when the verdict changes. + fn summary(&self) -> String { + if self.healthy { + return "healthy".to_string(); + } + if !self.error.is_empty() { + return format!("unhealthy: {}", self.error); + } + let detail = self + .unhealthy + .iter() + .map(|container| format!("{} is {}", container.name, container.status)) + .collect::<Vec<_>>() + .join(", "); + format!("unhealthy: {detail}") + } +} + +/// The cached verdict plus the loop that refreshes it. +pub(crate) struct HealthMonitor { + latest: RwLock<Verdict>, +} + +impl HealthMonitor { + /// Start the refresh loop and hand back the handle the RPC reads. + /// + /// Only called when the app opted in; an app that did not is never polled + /// by the gateway, so there is nothing to keep up to date. + pub(crate) fn spawn(status_file: Option<String>, runner: String) -> Arc<Self> { + match &status_file { + Some(path) => info!("app health is read from {path} every {REFRESH_INTERVAL:?}"), + None => { + info!("app health is judged from {runner} containers every {REFRESH_INTERVAL:?}") + } + } + let monitor = Arc::new(Self { + // The same rule the gateway applies to an instance that has + // registered but not yet answered a poll: not having run is not a + // pass. Registration happens long before the app is up. + // + // This seed is indistinguishable, over the wire, from a completed + // refresh that found the app unhealthy -- and the gateway believes + // an answered "no" immediately, without the hysteresis it applies + // to a failure to reach us. So a poll that lands in the window + // between this task starting and its first refresh finishing takes + // a healthy instance out of rotation for a round or two. The window + // is normally sub-second and only opens when the agent itself + // restarts, and it self-corrects; closing it properly means a third + // state ("starting") in `HealthResponse` that the gateway can run + // through its hysteresis, which is a protocol change, not a + // default. Erring the other way -- seeding `healthy` -- would let a + // genuinely broken app serve for a round on every agent restart, + // which is the failure this feature exists to prevent. + latest: RwLock::new(Verdict::unhealthy( + PENDING_NAME, + "health has not been determined yet".to_string(), + )), + }); + let task = monitor.clone(); + tokio::spawn(async move { + // Supervised, for the same reason the gateway's poller is: if this + // task ever died, `report()` would keep serving the last verdict + // forever and nothing in the response says how old it is, so an + // instance frozen on `healthy` would stay in rotation through any + // later failure. + loop { + let refresher = tokio::spawn(refresh_forever( + status_file.clone(), + runner.clone(), + task.clone(), + )); + match refresher.await { + Ok(()) => warn!("health refresh loop returned unexpectedly; restarting"), + Err(err) => warn!("health refresh loop died: {err}; restarting"), + } + tokio::time::sleep(REFRESH_INTERVAL).await; + } + }); + monitor + } + + /// The cached verdict. This is the whole cost of one `Worker.Health` call. + pub(crate) fn report(&self) -> Verdict { + self.latest + .read() + .or_panic("health monitor lock poisoned") + .clone() + } + + fn store(&self, verdict: Verdict) { + let mut latest = self.latest.write().or_panic("health monitor lock poisoned"); + if *latest != verdict { + info!("app health is now {}", verdict.summary()); + } + *latest = verdict; + } +} + +/// The refresh loop itself, in its own task so a panic in it is observable to +/// the supervisor above rather than ending health reporting silently. +async fn refresh_forever(status_file: Option<String>, runner: String, monitor: Arc<HealthMonitor>) { + let mut failures = 0; + loop { + match refresh(status_file.as_deref(), &runner).await { + Ok(verdict) => { + failures = 0; + monitor.store(verdict); + } + Err(err) => { + // Hold the previous verdict for a few rounds: a container being + // recreated underneath us looks exactly like this, and it + // resolves itself by the next tick. + failures += 1; + let error = format!("{err:#}"); + if failures < MAX_CONSECUTIVE_FAILURES { + debug!("health refresh failed ({failures}), keeping last verdict: {error}"); + } else { + // Once, on the way in. A runtime that stays down would + // otherwise be a warning every interval, forever -- + // `store` deduplicates the verdict it logs, and this has + // to do the same. + if failures == MAX_CONSECUTIVE_FAILURES { + warn!("health refresh has failed {failures} times: {error}"); + } + monitor.store(Verdict::failed(error)); + } + } + } + tokio::time::sleep(REFRESH_INTERVAL).await; + } +} + +/// One refresh, from whichever source the app declared. +/// +/// `Err` means "could not tell", which the caller tolerates for a few rounds. +/// A verdict of unhealthy is a different thing and comes back as `Ok`. +async fn refresh(status_file: Option<&str>, runner: &str) -> anyhow::Result<Verdict> { + let work = async { + match status_file { + Some(path) => Ok(read_health_file(path).await), + None => container_health::collect(runner).await, + } + }; + match tokio::time::timeout(REFRESH_TIMEOUT, work).await { + Ok(result) => result, + Err(_) => anyhow::bail!("health refresh timed out after {REFRESH_TIMEOUT:?}"), + } +} + +/// Read the app's own verdict out of the file it declared. +/// +/// Never returns `Err`: an unreadable or malformed file is the app failing to +/// hold up its end, which is a verdict, not an inability to reach one. +async fn read_health_file(path: &str) -> Verdict { + let contents = match read_bounded(path).await { + Ok(contents) => contents, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + return Verdict::unhealthy( + HEALTH_FILE_NAME, + format!("{} has not been written yet", sanitize(path)), + ) + } + Err(err) => { + // The path is measured into the compose hash, so echoing it tells + // an anonymous caller nothing it could not read there -- but it is + // still app-authored text on its way to a log line. + return Verdict::unhealthy( + HEALTH_FILE_NAME, + format!("cannot read {}: {err}", sanitize(path)), + ); + } + }; + judge_health_file(&contents, now_secs()) +} + +/// Open and read the health file, refusing anything that is not a plain file. +/// +/// Symlinks are followed. That is a decision, not an oversight: `O_NOFOLLOW` +/// only refuses a symlink as the *final* path component, so it never delivered +/// the property it looked like it delivered -- a container that can replace any +/// directory along the path redirects the read either way, and closing that +/// properly needs `openat2(RESOLVE_NO_SYMLINKS)` and a resolved-path check. +/// Buying the whole guarantee is not worth it here, because there is little +/// behind it: the read is bounded, and the contents are never echoed back (see +/// [`judge_health_file`]) -- what an app gets from redirecting it is the right +/// to lie about its own health, which it already has, since it writes the file. +/// Meanwhile refusing the last component alone breaks the ordinary case of a +/// health file living under a symlinked data directory. +/// +/// The regular-file check stays, and is load-bearing for a different reason. A +/// FIFO is worse than it looks: `tokio::fs` opens on the blocking pool, and a +/// blocking task cannot be cancelled, so `open(2)` on a reader-less FIFO parks +/// that thread forever while the outer timeout merely abandons the future. One +/// thread leaks per refresh, silently -- the systemd watchdog probes over HTTP +/// and never touches the pool -- until the agent can no longer do any +/// filesystem work at all. `O_NONBLOCK` makes that open return immediately +/// instead, and the file-type check rejects what it opened. +async fn read_bounded(path: &str) -> std::io::Result<String> { + let path = path.to_string(); + tokio::task::spawn_blocking(move || { + use std::io::Read as _; + use std::os::unix::fs::OpenOptionsExt as _; + + let file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NONBLOCK) + .open(&path)?; + let kind = file.metadata()?.file_type(); + if !kind.is_file() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "health file is not a regular file", + )); + } + let mut contents = String::new(); + file.take(MAX_HEALTH_FILE_BYTES) + .read_to_string(&mut contents)?; + Ok(contents) + }) + .await + .unwrap_or_else(|err| Err(std::io::Error::other(err))) +} + +/// Apply the two-line contract to what the file actually contained. +/// +/// Split out so the rule can be tested without a filesystem, and so the +/// staleness bound is tested against an injected clock rather than a sleep. +fn judge_health_file(contents: &str, now: u64) -> Verdict { + let mut lines = contents.lines(); + let Some(state) = lines.next().map(str::trim) else { + return Verdict::unhealthy(HEALTH_FILE_NAME, "health file is empty".to_string()); + }; + let Some(timestamp) = lines.next().map(str::trim) else { + return Verdict::unhealthy( + HEALTH_FILE_NAME, + "health file must have two lines: state and unix timestamp".to_string(), + ); + }; + let Ok(written) = timestamp.parse::<u64>() else { + // Deliberately not quoting the line. The report is served to anonymous + // callers on the external listener, and the bytes on line 2 are + // whatever was at that path -- which, if something replaced the file, + // is not what the app wrote and not what was measured. + return Verdict::unhealthy( + HEALTH_FILE_NAME, + "health file line 2 is not a unix time in seconds".to_string(), + ); + }; + // A clock that ran backwards, or an app stamping slightly ahead, is not + // evidence of anything. Only age counts. + let age = now.saturating_sub(written); + if age > HEALTH_FILE_MAX_AGE_SECS { + return Verdict::unhealthy( + HEALTH_FILE_NAME, + format!( + "health file is stale: written {age}s ago, limit is {HEALTH_FILE_MAX_AGE_SECS}s" + ), + ); + } + match state.to_ascii_lowercase().as_str() { + "healthy" => Verdict::healthy(), + "unhealthy" => Verdict::unhealthy(HEALTH_FILE_NAME, "app reports unhealthy".to_string()), + // Anything else is the app saying something this agent does not + // understand, which is not a pass. Not quoted, for the same reason as + // the timestamp above. + _ => Verdict::unhealthy( + HEALTH_FILE_NAME, + "health file line 1 is neither healthy nor unhealthy".to_string(), + ), + } +} + +/// Make app-authored text safe to put in a report. +/// +/// The report is served to anonymous callers on the external listener and is +/// logged by the gateway, so control characters -- newlines that forge a log +/// line, escapes that repaint a terminal, bidi overrides that reverse what a +/// reader sees -- must not survive, and the length must be bounded. The gateway +/// does the same again on receipt, because it does not trust this agent to have +/// done it. +fn sanitize(token: &str) -> String { + let cleaned = dstack_types::sanitize_for_log(token.trim(), MAX_REPORTED_BYTES); + if cleaned.is_empty() { + return "<empty>".to_string(); + } + cleaned +} + +fn now_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|since| since.as_secs()) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + const NOW: u64 = 1_771_234_567; + + fn file(contents: &str) -> Verdict { + judge_health_file(contents, NOW) + } + + fn reason(verdict: &Verdict) -> String { + verdict + .unhealthy + .first() + .map(|container| container.status.clone()) + .unwrap_or_default() + } + + #[test] + fn a_fresh_healthy_file_is_healthy() { + let verdict = file(&format!("healthy\n{NOW}\n")); + assert_eq!(verdict, Verdict::healthy()); + } + + #[test] + fn the_state_is_case_insensitive_and_whitespace_tolerant() { + assert!(file(&format!(" HEALTHY \n {NOW} \n")).healthy); + } + + #[test] + fn a_fresh_unhealthy_file_is_unhealthy() { + let verdict = file(&format!("unhealthy\n{NOW}")); + assert!(!verdict.healthy); + assert_eq!(reason(&verdict), "app reports unhealthy"); + } + + /// The whole point of the timestamp: an app that stopped updating the file + /// must not keep its instance in rotation on the strength of the last thing + /// it wrote. + #[test] + fn a_stale_healthy_file_is_not_healthy() { + let written = NOW - HEALTH_FILE_MAX_AGE_SECS - 1; + let verdict = file(&format!("healthy\n{written}")); + assert!(!verdict.healthy); + assert!( + reason(&verdict).contains("stale"), + "unexpected reason: {}", + reason(&verdict) + ); + } + + #[test] + fn a_file_written_exactly_at_the_limit_is_still_fresh() { + let written = NOW - HEALTH_FILE_MAX_AGE_SECS; + assert!(file(&format!("healthy\n{written}")).healthy); + } + + /// A clock that jumped, or an app stamping a moment ahead, is not a failure. + #[test] + fn a_timestamp_in_the_future_is_treated_as_fresh() { + assert!(file(&format!("healthy\n{}", NOW + 3600)).healthy); + } + + #[test] + fn a_missing_timestamp_line_is_not_healthy() { + let verdict = file("healthy\n"); + assert!(!verdict.healthy); + assert!(reason(&verdict).contains("two lines")); + } + + #[test] + fn an_unparseable_timestamp_is_not_healthy() { + let verdict = file("healthy\nyesterday"); + assert!(!verdict.healthy); + assert!(reason(&verdict).contains("unix time")); + } + + /// The report goes to anonymous callers on the external listener, and the + /// bytes at that path are not the bytes that were measured -- a symlink or + /// a replaced file makes them something else entirely. Naming the rule that + /// failed is enough to debug with; quoting the line is an exfiltration + /// channel. + #[test] + fn a_malformed_file_is_never_quoted_back() { + let secret = "AKIAIOSFODNN7EXAMPLE"; + let verdict = file(&format!("healthy\n{secret}")); + assert!(!reason(&verdict).contains(secret), "{}", reason(&verdict)); + + let verdict = file(&format!("{secret}\n{NOW}")); + assert!(!reason(&verdict).contains(secret), "{}", reason(&verdict)); + } + + #[test] + fn an_empty_file_is_not_healthy() { + let verdict = file(""); + assert!(!verdict.healthy); + assert!(reason(&verdict).contains("empty")); + } + + /// Unknown states hold traffic back rather than pass, the same way an + /// unknown container health string does. + #[test] + fn an_unrecognized_state_is_not_healthy() { + let verdict = file(&format!("degraded\n{NOW}")); + assert!(!verdict.healthy); + assert!(reason(&verdict).contains("neither healthy nor unhealthy")); + } + + /// The `health_status_file` path is app-authored and reaches both an anonymous + /// caller and a log line, so it is bounded and stripped like anything else + /// that crosses that boundary. + #[tokio::test] + async fn a_hostile_path_cannot_forge_a_log_line() { + let forged = "/nonexistent/\u{1b}[31mx\nlevel=fatal msg=owned"; + let verdict = read_health_file(forged).await; + let reason = reason(&verdict); + assert!(!reason.contains('\n'), "newline survived: {reason:?}"); + assert!(!reason.contains('\u{1b}'), "escape survived: {reason:?}"); + } + + #[tokio::test] + async fn a_very_long_path_is_truncated() { + let long = format!("/nonexistent/{}", "x".repeat(MAX_REPORTED_BYTES * 4)); + let verdict = read_health_file(&long).await; + assert!(reason(&verdict).len() < MAX_REPORTED_BYTES * 2); + } + + /// A health file under a symlinked data directory is the ordinary case, not + /// an attack, and refusing it was breaking working apps for nothing: + /// `O_NOFOLLOW` only ever refused a symlink as the *final* path component, + /// so anything that could swap a parent directory redirected the read + /// regardless. What is left behind the guarantee is small -- the read is + /// bounded and the contents are never echoed back -- so all an app buys by + /// redirecting it is the right to lie about its own health, which it + /// already has, since it writes the file. + #[tokio::test] + async fn a_symlinked_health_file_is_followed() { + let dir = tempfile::tempdir().expect("tempdir"); + let target = dir.path().join("target"); + std::fs::write(&target, format!("healthy\n{}\n", now_secs())).expect("write"); + assert!( + read_health_file(target.to_str().expect("utf8")) + .await + .healthy, + "the target must read healthy, or this test proves nothing" + ); + + let link = dir.path().join("health"); + std::os::unix::fs::symlink(&target, &link).expect("symlink"); + let verdict = read_health_file(link.to_str().expect("utf8")).await; + assert!( + verdict.healthy, + "the link should read through to its target: {verdict:?}" + ); + } + + /// `tokio::fs` opens on the blocking pool and a blocking task cannot be + /// cancelled, so a reader-less FIFO would park a thread forever -- one per + /// refresh, until the agent can do no filesystem work at all. `O_NONBLOCK` + /// makes the open fail instead. This test hangs if that flag is dropped. + #[tokio::test] + async fn a_fifo_health_file_does_not_park_a_thread() { + let dir = tempfile::tempdir().expect("tempdir"); + let fifo = dir.path().join("health"); + let path = std::ffi::CString::new(fifo.to_str().expect("utf8")).expect("cstring"); + assert_eq!(unsafe { libc::mkfifo(path.as_ptr(), 0o600) }, 0); + + let verdict = tokio::time::timeout( + Duration::from_secs(5), + read_health_file(fifo.to_str().expect("utf8")), + ) + .await + .expect("opening a FIFO must not block"); + assert!(!verdict.healthy, "{verdict:?}"); + } + + #[tokio::test] + async fn a_directory_is_not_a_health_file() { + let dir = tempfile::tempdir().expect("tempdir"); + let verdict = read_health_file(dir.path().to_str().expect("utf8")).await; + assert!(!verdict.healthy, "{verdict:?}"); + } + + #[tokio::test] + async fn a_missing_health_file_is_not_healthy() { + let verdict = read_health_file("/nonexistent/dstack-health-test").await; + assert!(!verdict.healthy); + assert!(reason(&verdict).contains("has not been written yet")); + } + + #[tokio::test] + async fn a_health_file_is_read_from_disk() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("health"); + std::fs::write(&path, format!("healthy\n{}\n", now_secs())).expect("write"); + let verdict = read_health_file(path.to_str().expect("utf8")).await; + assert!(verdict.healthy, "{verdict:?}"); + } + + /// An app cannot make the agent allocate an arbitrary amount by pointing + /// `health_status_file` at something huge. + #[tokio::test] + async fn an_oversized_health_file_is_bounded() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("health"); + let mut contents = format!("healthy\n{}\n", now_secs()); + contents.push_str(&"padding\n".repeat(4096)); + std::fs::write(&path, &contents).expect("write"); + let read = read_bounded(path.to_str().expect("utf8")) + .await + .expect("read"); + assert!(read.len() as u64 <= MAX_HEALTH_FILE_BYTES); + // The two lines that matter are inside the bound, so the verdict is + // still reached. + assert!(judge_health_file(&read, now_secs()).healthy); + } + + /// The monitor must not start out vouching for an app that has not been + /// looked at yet. + #[tokio::test] + async fn the_first_report_is_not_healthy() { + let monitor = HealthMonitor::spawn( + Some("/nonexistent/dstack-health-test".to_string()), + "docker-compose".to_string(), + ); + let verdict = monitor.report(); + assert!(!verdict.healthy, "{verdict:?}"); + } +} diff --git a/dstack/guest-agent/src/http_routes.rs b/dstack/guest-agent/src/http_routes.rs index c8fa44df2..3d0f662f4 100644 --- a/dstack/guest-agent/src/http_routes.rs +++ b/dstack/guest-agent/src/http_routes.rs @@ -9,7 +9,7 @@ use crate::guest_api_service::{list_containers, GuestApiHandler}; use crate::rpc_service::{AppState, ExternalRpcHandler}; use anyhow::Result; use docker_logs::parse_duration; -use dstack_guest_agent_rpc::{worker_server::WorkerRpc, AppInfo}; +use dstack_guest_agent_rpc::v0::{worker_server::WorkerRpc, AppInfo}; use guest_api::guest_api_server::GuestApiRpc; use ra_rpc::{CallContext, RpcCall}; use rinja::Template; @@ -58,6 +58,13 @@ async fn index(state: &State<AppState>) -> Result<RawHtml<String>, String> { let handler = GuestApiHandler::construct(context) .map_err(|e| format!("Failed to construct RPC handler: {}", e))?; let system_info = handler.sys_info().await.unwrap_or_default(); + // Only sampled when the page will actually render it. Asking otherwise + // would spawn a collector on guests that keep their sysinfo private. + let gpu_info = if public_sysinfo { + crate::gpu_info::gpu_info() + } else { + Default::default() + }; let containers = list_containers().await.unwrap_or_default().containers; let model = crate::models::Dashboard { @@ -74,6 +81,7 @@ async fn index(state: &State<AppState>) -> Result<RawHtml<String>, String> { public_tcbinfo, cloud_vendor, cloud_product, + gpu_info, }; match model.render() { Ok(html) => Ok(RawHtml(html)), @@ -93,7 +101,11 @@ async fn metrics(state: &State<AppState>) -> Result<String, String> { .map_err(|e| format!("Failed to construct RPC handler: {}", e))?; let system_info = handler.sys_info().await.unwrap_or_default(); - let model = crate::models::Metrics { system_info }; + let gpu_info = crate::gpu_info::gpu_info(); + let model = crate::models::Metrics { + system_info, + gpu_info, + }; match model.render() { Ok(body) => Ok(body), Err(err) => Err(format!("Failed to render template: {err}")), diff --git a/dstack/guest-agent/src/lib.rs b/dstack/guest-agent/src/lib.rs index ef6f844ce..7c70e192a 100644 --- a/dstack/guest-agent/src/lib.rs +++ b/dstack/guest-agent/src/lib.rs @@ -3,18 +3,19 @@ // SPDX-License-Identifier: Apache-2.0 pub const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION"); -pub const GIT_REV: &str = git_version::git_version!( - args = ["--abbrev=20", "--always", "--dirty=-modified"], - prefix = "git:", - fallback = "unknown" -); +pub const GIT_REV: &str = dstack_build_info::git_revision!(); pub mod backend; pub mod config; +mod container_health; +mod gpu_attest; +mod gpu_info; mod guest_api_service; +mod health; mod http_routes; mod models; pub mod rpc_service; +pub mod rpc_service_v1; mod server; mod socket_activation; diff --git a/dstack/guest-agent/src/models.rs b/dstack/guest-agent/src/models.rs index a50cf340d..c8719105b 100644 --- a/dstack/guest-agent/src/models.rs +++ b/dstack/guest-agent/src/models.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -use guest_api::{Container, SystemInfo}; +use guest_api::{Container, GpuInfoResponse, SystemInfo}; use rinja::Template; mod filters { @@ -32,6 +32,77 @@ mod filters { pub fn hex(s: &[u8]) -> Result<String, rinja::Error> { Ok(hex::encode(s)) } + + /// Renders a second count as `1d 2h 3m 4s`. `SystemInfo.uptime` is a + /// `uint64` of seconds and the page printed it raw, so a guest one minute + /// into its life displayed `68` with no unit at all. + /// + /// Display only: `dstack_guest_uptime_seconds` keeps the raw count, which + /// is what a Prometheus gauge of that name has to carry. + pub fn hduration(s: &u64) -> Result<String, rinja::Error> { + let (days, hours) = (s / 86_400, (s % 86_400) / 3_600); + let (minutes, seconds) = ((s % 3_600) / 60, s % 60); + let mut out = String::new(); + if days > 0 { + out.push_str(&format!("{days}d ")); + } + if days > 0 || hours > 0 { + out.push_str(&format!("{hours}h ")); + } + if days > 0 || hours > 0 || minutes > 0 { + out.push_str(&format!("{minutes}m ")); + } + out.push_str(&format!("{seconds}s")); + Ok(out) + } + + /// Recovers a load average from the fixed-point `u32` the wire carries. + /// + /// `SystemInfo.loadavg_*` is the load times 100, so every consumer has to + /// divide it back. The dashboard did divide, then appended a `%`; a load + /// average is a count of runnable tasks, not a percentage, and `1.00` on a + /// single-core guest means saturated rather than one percent. `/metrics` + /// did not divide at all, so a load of 0.40 was published as `40`. + pub fn load(s: &u32) -> Result<String, rinja::Error> { + Ok(format!("{:.2}", f64::from(*s) / 100.0)) + } + + /// Drops a zero PCI domain. NVML reports `00000000:01:00.0` where lspci and + /// the kernel print `0000:01:00.0` or just `01:00.0`. + /// + /// Only a zero domain is dropped, and only from the three-field form: a + /// non-zero domain distinguishes two cards that would otherwise display the + /// same address, and `00:00.0` is a bus, not a domain. + /// + /// Display only. `/metrics` keeps the full string, because that is what + /// host-side tooling joins on. + pub fn short_bdf(s: &str) -> Result<&str, rinja::Error> { + let Some((domain, rest)) = s.split_once(':') else { + return Ok(s); + }; + if rest.contains(':') && !domain.is_empty() && domain.bytes().all(|b| b == b'0') { + return Ok(rest); + } + Ok(s) + } + + /// The label set every `dstack_gpu_*` series carries, matching + /// dcgm-exporter so host-side tooling that speaks BDF can correlate. + /// Written once here rather than repeated in the template for each metric. + pub fn gpu_labels(gpu: &guest_api::GpuDevice) -> Result<String, rinja::Error> { + Ok(format!( + "{{index=\"{}\", uuid=\"{}\", pci_bus_id=\"{}\"}}", + gpu.index, + prometheus_label(&gpu.uuid)?, + prometheus_label(&gpu.pci_bus_id)?, + )) + } + + pub fn prometheus_label(s: &str) -> Result<String, rinja::Error> { + Ok(s.replace('\\', "\\\\") + .replace('\n', "\\n") + .replace('"', "\\\"")) + } } #[derive(Template)] @@ -50,10 +121,297 @@ pub struct Dashboard { pub public_tcbinfo: bool, pub cloud_vendor: String, pub cloud_product: String, + pub gpu_info: GpuInfoResponse, } #[derive(Template)] #[template(path = "metrics.tpl", escape = "none")] pub struct Metrics { pub system_info: SystemInfo, + pub gpu_info: GpuInfoResponse, +} + +#[cfg(test)] +mod tests { + use super::*; + use guest_api::GpuDevice; + use rinja::Template; + + fn render(gpu_info: GpuInfoResponse) -> String { + Metrics { + system_info: Default::default(), + gpu_info, + } + .render() + .expect("render") + } + + fn gpu(index: u32) -> GpuDevice { + GpuDevice { + index, + uuid: format!("GPU-{index}"), + pci_bus_id: format!("00000000:0{index}:00.0"), + ..Default::default() + } + } + + /// The regression this whole design exists for: a sample older than the + /// collection TTL is still exposed. Emitting nothing meant every scrape at + /// Prometheus' default interval saw zero GPU series on a healthy card. + #[test] + fn a_stale_sample_still_produces_series() { + let body = render(GpuInfoResponse { + gpus: vec![GpuDevice { + utilization_gpu: Some(37), + ..gpu(0) + }], + sample_age_ms: Some(60_000), + ..Default::default() + }); + assert!(body.contains( + r#"dstack_gpu_utilization_percent{index="0", uuid="GPU-0", pci_bus_id="00000000:00:00.0"} 37"# + )); + assert!(body.contains("dstack_gpu_nvml_up 1")); + assert!(body.contains("dstack_gpu_sample_age_seconds 60")); + } + + /// A field NVML could not answer must be absent, not zero: a scraped 0 is + /// indistinguishable from an idle GPU. + #[test] + fn a_field_that_failed_to_sample_emits_no_series() { + let body = render(GpuInfoResponse { + gpus: vec![GpuDevice { + utilization_gpu: None, + temperature_c: Some(0), + ..gpu(0) + }], + ..Default::default() + }); + assert!(!body.contains("dstack_gpu_utilization_percent{")); + assert!(body.contains(r#"dstack_gpu_temperature_celsius{index="0""#)); + } + + /// Failed queries are counted from a list. When they were a `"; "`-joined + /// string, a message containing that separator inflated the count. + #[test] + fn query_errors_counts_entries_not_separators() { + let body = render(GpuInfoResponse { + gpus: vec![GpuDevice { + errors: vec![ + "power: not supported".into(), + "memory: unknown error; retry advised".into(), + ], + ..gpu(0) + }], + ..Default::default() + }); + assert!(body.contains( + r#"dstack_gpu_query_errors{index="0", uuid="GPU-0", pci_bus_id="00000000:00:00.0"} 2"# + )); + } + + /// A guest with no GPU reports `nvml_up 1` with no devices: the collector + /// ran and found nothing. `nvml_up 0` is reserved for "could not tell". + #[test] + fn a_guest_without_gpus_is_not_reported_as_a_collection_failure() { + let body = render(GpuInfoResponse::default()); + assert!(body.contains("dstack_gpu_nvml_up 1")); + assert!(!body.contains("dstack_gpu_utilization_percent{")); + assert!(!body.contains("dstack_gpu_cc_ready")); + + let failed = render(GpuInfoResponse { + error: "NVIDIA driver is not loaded".into(), + ..Default::default() + }); + assert!(failed.contains("dstack_gpu_nvml_up 0")); + } + + /// Label values are escaped, so a hostile UUID cannot inject a series. + #[test] + fn label_values_are_escaped() { + let body = render(GpuInfoResponse { + gpus: vec![GpuDevice { + uuid: r#"GPU-"injected" \ end"#.into(), + utilization_gpu: Some(1), + ..gpu(0) + }], + ..Default::default() + }); + assert!(body.contains(r#"uuid="GPU-\"injected\" \\ end""#), "{body}"); + } + + /// Watts are rendered from milliwatts. Raw f32 Display would print + /// 70.123001 for a card drawing 70.123 W. + #[test] + fn dashboard_power_is_formatted_to_one_decimal() { + let dashboard = Dashboard { + app_name: String::new(), + app_id: vec![], + instance_id: vec![], + device_id: vec![], + key_provider_info: String::new(), + tcb_info: String::new(), + containers: vec![], + system_info: Default::default(), + public_sysinfo: true, + public_logs: false, + public_tcbinfo: false, + cloud_vendor: String::new(), + cloud_product: String::new(), + gpu_info: GpuInfoResponse { + gpus: vec![GpuDevice { + power_usage_mw: Some(70_123), + ..gpu(0) + }], + cc_enabled: Some(true), + sample_age_ms: Some(2_500), + ..Default::default() + }, + }; + let html = dashboard.render().expect("render"); + assert!(html.contains("70.1 W"), "{html}"); + } + + /// The CC fields are labelled rows, so the value carries no prose of its + /// own. What must not come back is the old run-on line, and the age has to + /// round like every other number on the page. + #[test] + fn dashboard_renders_the_gpu_header_as_labelled_rows() { + let html = dashboard_with(GpuInfoResponse { + gpus: vec![gpu(0)], + cc_enabled: Some(true), + cc_ready: Some(false), + sample_age_ms: Some(1_049), + ..Default::default() + }); + assert!(html.contains(">Confidential Computing<"), "{html}"); + assert!(html.contains(">true<"), "{html}"); + assert!(html.contains(">GPU Ready State<"), "{html}"); + assert!(html.contains(">false<"), "{html}"); + assert!(html.contains("1.0 s"), "{html}"); + assert!(!html.contains("enabled = true"), "{html}"); + } + + /// An unset field is a third state, not a false. Reporting a GPU whose CC + /// status could not be read as `false` would invert the claim. + #[test] + fn dashboard_distinguishes_unknown_cc_state_from_false() { + let html = dashboard_with(GpuInfoResponse { + gpus: vec![gpu(0)], + cc_enabled: None, + cc_ready: None, + ..Default::default() + }); + assert!(html.contains(">unknown<"), "{html}"); + assert!(!html.contains(">false<"), "{html}"); + } + + /// Every other block on the page is a card. A bare paragraph on the page + /// background is what this section used to render into. + #[test] + fn dashboard_keeps_the_gpu_section_inside_a_card() { + let html = dashboard_with(GpuInfoResponse::default()); + assert!( + html.contains(r#"<div class="info-section">No NVIDIA GPUs</div>"#), + "{html}" + ); + } + + /// The zero domain is display noise, but only the noise may go. A non-zero + /// domain is what tells two cards on a multi-domain host apart, and + /// `00:00.0` is bus zero, not a domain that can be dropped. + #[test] + fn the_pci_domain_is_dropped_only_where_it_carries_nothing() { + use super::filters::short_bdf; + + assert_eq!(short_bdf("00000000:01:00.0").unwrap(), "01:00.0"); + assert_eq!(short_bdf("0000:01:00.0").unwrap(), "01:00.0"); + assert_eq!(short_bdf("00010000:01:00.0").unwrap(), "00010000:01:00.0"); + assert_eq!(short_bdf("00:00.0").unwrap(), "00:00.0"); + assert_eq!(short_bdf("01:00.0").unwrap(), "01:00.0"); + } + + /// The dashboard drops the UUID column and shortens the address. Neither + /// may reach `/metrics`: those labels are what a host-side exporter joins + /// on, so they carry the identifiers verbatim. + #[test] + fn metrics_labels_keep_the_full_identifiers() { + let body = render(GpuInfoResponse { + gpus: vec![GpuDevice { + uuid: "GPU-b2880e21-86ab".into(), + pci_bus_id: "00000000:01:00.0".into(), + ..gpu(0) + }], + ..Default::default() + }); + assert!(body.contains(r#"uuid="GPU-b2880e21-86ab""#), "{body}"); + assert!(body.contains(r#"pci_bus_id="00000000:01:00.0""#), "{body}"); + } + + /// A bare `68` on the page said neither seconds nor minutes. The unit has + /// to survive every magnitude, including the boundary where a field first + /// becomes non-zero. + #[test] + fn uptime_is_rendered_with_units() { + use super::filters::hduration; + + assert_eq!(hduration(&0).unwrap(), "0s"); + assert_eq!(hduration(&68).unwrap(), "1m 8s"); + assert_eq!(hduration(&3_600).unwrap(), "1h 0m 0s"); + assert_eq!(hduration(&90_061).unwrap(), "1d 1h 1m 1s"); + } + + /// The wire carries the load times 100. Both consumers got it wrong in + /// opposite directions: the page divided and then called the result a + /// percentage, `/metrics` published the undivided integer. + #[test] + fn load_average_is_neither_scaled_nor_a_percentage() { + use super::filters::load; + + assert_eq!(load(&40).unwrap(), "0.40"); + assert_eq!(load(&100).unwrap(), "1.00"); + assert_eq!(load(&1_234).unwrap(), "12.34"); + } + + /// A gauge named `load1` reporting 40 for a load of 0.40 is off by two + /// orders of magnitude, which is worse than the cosmetic `%` on the page. + #[test] + fn metrics_publish_load_averages_unscaled() { + let body = Metrics { + system_info: SystemInfo { + loadavg_one: 40, + loadavg_five: 100, + loadavg_fifteen: 1_234, + ..Default::default() + }, + gpu_info: Default::default(), + } + .render() + .expect("render"); + assert!(body.contains("dstack_guest_load1 0.40"), "{body}"); + assert!(body.contains("dstack_guest_load5 1.00"), "{body}"); + assert!(body.contains("dstack_guest_load15 12.34"), "{body}"); + assert!(body.contains("system_load_average_1m 0.40"), "{body}"); + } + + fn dashboard_with(gpu_info: GpuInfoResponse) -> String { + Dashboard { + app_name: String::new(), + app_id: vec![], + instance_id: vec![], + device_id: vec![], + key_provider_info: String::new(), + tcb_info: String::new(), + containers: vec![], + system_info: Default::default(), + public_sysinfo: true, + public_logs: false, + public_tcbinfo: false, + cloud_vendor: String::new(), + cloud_product: String::new(), + gpu_info, + } + .render() + .expect("render") + } } diff --git a/dstack/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs index cb9680353..7d0159cd0 100644 --- a/dstack/guest-agent/src/rpc_service.rs +++ b/dstack/guest-agent/src/rpc_service.rs @@ -2,30 +2,31 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::{Duration, Instant}; use anyhow::{Context, Result}; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; use cert_client::CertRequestClient; -use dstack_guest_agent_rpc::{ +use dstack_attest::default_verifier; +use dstack_guest_agent_rpc::v0::{ dstack_guest_server::{DstackGuestRpc, DstackGuestServer}, tappd_server::{TappdRpc, TappdServer}, worker_server::{WorkerRpc, WorkerServer}, - AppInfo, AttestResponse, DeriveK256KeyResponse, DeriveKeyArgs, GetAttestationForAppKeyRequest, - GetKeyArgs, GetKeyResponse, GetQuoteResponse, GetTlsKeyArgs, GetTlsKeyResponse, RawQuoteArgs, - SignRequest, SignResponse, TdxQuoteArgs, TdxQuoteResponse, VerifyRequest, VerifyResponse, - WorkerVersion, + AppInfo, AttestResponse, DeriveK256KeyResponse, DeriveKeyArgs, EmitEventArgs, + GetAttestationForAppKeyRequest, GetKeyArgs, GetKeyResponse, GetQuoteResponse, GetTlsKeyArgs, + GetTlsKeyResponse, RawQuoteArgs, SignRequest, SignResponse, TdxQuoteArgs, TdxQuoteResponse, + VerifyRequest, VerifyResponse, WorkerVersion, }; use dstack_types::{AppKeys, SysConfig}; use ed25519_dalek::ed25519::signature::hazmat::{PrehashSigner, PrehashVerifier}; -use ed25519_dalek::{ - Signer as Ed25519Signer, SigningKey as Ed25519SigningKey, Verifier as Ed25519Verifier, -}; +use ed25519_dalek::{Signer as Ed25519Signer, SigningKey as Ed25519SigningKey, Verifier}; use fs_err as fs; use k256::ecdsa::SigningKey; use or_panic::ResultOrPanic; use ra_rpc::{CallContext, RpcCall}; use ra_tls::{ + api_v1::sign_recoverable_keccak256, attestation::{ QuoteContentType, TdxAttestationExt, VersionedAttestation, DEFAULT_HASH_ALGORITHM, }, @@ -35,7 +36,7 @@ use ra_tls::{ use rcgen::KeyPair; use ring::rand::{SecureRandom, SystemRandom}; use serde_json::json; -use sha3::{Digest, Keccak256}; + use tracing::error; use crate::{ @@ -43,6 +44,39 @@ use crate::{ config::Config, }; +/// How long a failed identity decode is left alone before another call is +/// allowed to touch the platform again. +/// +/// The cache makes the happy path free, but nothing caches a failure, and the +/// retry path is reachable from `/prpc/v1/Info` -- anonymous, publicly +/// reachable, and one hardware quote plus an event-log replay per attempt, +/// under the global quote lock. Retrying per call would hand any caller that +/// can route to the CVM exactly the lever the cache exists to remove, for as +/// long as the platform stays broken. A floor of half a minute bounds that to +/// one attempt per interval while still letting a platform that was only +/// momentarily unable to attest recover on its own. +const IDENTITY_RETRY_INTERVAL: Duration = Duration::from_secs(30); + +/// Decode the immutable identity fields out of a boot attestation. +/// +/// Costs a quote and an event-log replay, which is why its result is cached for +/// the life of the process. +fn decode_identity(inner: &AppStateInner) -> Result<AppIdentity> { + let attestation = inner.info_attestation()?.into_v1(); + let app_info = attestation + .decode_app_info(false) + .context("failed to decode app info")?; + Ok(AppIdentity { + app_id: app_info.app_id, + instance_id: app_info.instance_id, + device_id: app_info.device_id, + mr_aggregated: app_info.mr_aggregated.to_vec(), + os_image_hash: app_info.os_image_hash, + compose_hash: app_info.compose_hash, + key_provider_info: String::from_utf8(app_info.key_provider_info).unwrap_or_default(), + }) +} + fn read_dmi_file(name: &str) -> String { fs::read_to_string(format!("/sys/class/dmi/id/{name}")) .map(|s| s.trim().to_string()) @@ -61,6 +95,47 @@ struct AppStateInner { cert_client: CertRequestClient, demo_cert: RwLock<String>, platform: Arc<dyn PlatformBackend>, + /// Present only when the app opted into health gating; see `health`. + health: Option<Arc<crate::health::HealthMonitor>>, + /// Serialises on-demand GPU attestation. + gpu_attestor: crate::gpu_attest::GpuAttestor, + /// The app root key, parsed once. + /// + /// `None` when the key provider handed us something that is not a valid + /// secp256k1 scalar. Kept non-fatal so this cannot turn a boot problem into + /// a boot failure -- the RPCs that need it report it per call, as they did + /// when each parsed the key itself. + app_root_signing_key: Option<SigningKey>, + /// Identity as decoded from the boot attestation. See [`AppIdentity`]. + identity: RwLock<Option<Arc<AppIdentity>>>, + /// When the last identity decode failed, so the retry can be throttled. + /// See [`IDENTITY_RETRY_INTERVAL`]. Separate from the cache above because + /// it is written only on the degraded path, and read only when the cache + /// is empty. + identity_last_failure: Mutex<Option<Instant>>, + /// `sys_vendor` and `product_name`, read once. Neither changes while the + /// VM is running. + cloud_vendor: String, + cloud_product: String, +} + +/// The identity fields v1 `Info` reports, decoded once. +/// +/// Every one of these is fixed for the life of the VM: they come out of the +/// launch measurements. Recomputing them per call meant generating a fresh +/// hardware quote and replaying the RTMR event log under the global quote lock +/// on every `Info` -- including anonymous calls to the public `/prpc/v1/Info`, +/// which let any caller that can route to the CVM monopolise that lock and +/// starve the attestation path the agent actually needs. +#[derive(Debug)] +pub(crate) struct AppIdentity { + pub(crate) app_id: Vec<u8>, + pub(crate) instance_id: Vec<u8>, + pub(crate) device_id: Vec<u8>, + pub(crate) mr_aggregated: Vec<u8>, + pub(crate) os_image_hash: Vec<u8>, + pub(crate) compose_hash: Vec<u8>, + pub(crate) key_provider_info: String, } impl AppStateInner { @@ -68,12 +143,18 @@ impl AppStateInner { self.platform.attestation_for_info() } - async fn issue_cert(&self, key: &KeyPair, config: CertConfigV2) -> Result<Vec<String>> { + async fn issue_cert( + &self, + key: &KeyPair, + config: CertConfigV2, + wire: AttestationWire, + ) -> Result<Vec<String>> { let pubkey = key.public_key_der(); - let attestation = self - .platform - .certificate_attestation(&pubkey) - .context("Failed to get certificate attestation")?; + let attestation = wire.apply( + self.platform + .certificate_attestation(&pubkey) + .context("Failed to get certificate attestation")?, + ); let csr = CertSigningRequestV2 { confirm: "please sign cert:".to_string(), pubkey, @@ -103,6 +184,8 @@ impl AppStateInner { not_after: None, not_before: None, }, + // Served only by the frozen v0 `Info` (`app_cert`). + AttestationWire::Legacy, ) .await .context("Failed to get app cert")? @@ -143,11 +226,38 @@ impl AppState { let sys_config: SysConfig = serde_json::from_str(&fs::read_to_string(&config.sys_config_file)?) .context("Failed to parse VM config")?; + let collateral_urls = sys_config.collateral_urls(); + let gpu_attestor = crate::gpu_attest::GpuAttestor::new(); let vm_config = sys_config.vm_config; - let cert_client = - CertRequestClient::create(&keys, config.pccs_url.as_deref(), vm_config.clone()) - .await - .context("Failed to create cert signer")?; + // Same trust anchor decision as dstack-util: never host-supplied, and + // development roots only when this guest published them itself. + let verifier = Arc::new(default_verifier(&collateral_urls)?); + let cert_client = CertRequestClient::create(&keys, verifier, vm_config.clone()) + .await + .context("Failed to create cert signer")?; + // Only run the refresh loop when the app asked the gateway to gate on + // its health. Nothing polls an app that did not, so recomputing a + // verdict nobody reads would be pure cost inside the CVM. + let health = config + .app_compose + .requirements + .as_ref() + .filter(|requirements| requirements.health_check) + .map(|requirements| { + crate::health::HealthMonitor::spawn( + requirements.health_status_file.clone(), + config.app_compose.runner.clone(), + ) + }); + // Parsed once, and non-fatally: an unusable app root key is reported by + // the RPCs that need one, not by refusing to start. + let app_root_signing_key = match SigningKey::from_slice(&keys.k256_key) { + Ok(key) => Some(key), + Err(err) => { + error!("the app root k256 key did not parse: {err:?}"); + None + } + }; let me = Self { inner: Arc::new(AppStateInner { config, @@ -156,8 +266,23 @@ impl AppState { demo_cert: RwLock::new(String::new()), vm_config, platform, + health, + gpu_attestor, + app_root_signing_key, + identity: RwLock::new(None), + identity_last_failure: Mutex::new(None), + cloud_vendor: read_dmi_file("sys_vendor"), + cloud_product: read_dmi_file("product_name"), }), }; + // Decode identity now so no request has to. Non-fatal: a platform that + // cannot attest at this instant would otherwise take the whole agent + // down, and `identity()` retries on demand. A failure here counts as + // the first attempt and arms the retry throttle, so the boot attempt + // and a request-driven one are on the same budget. + if let Err(err) = me.identity() { + error!("failed to decode app identity at startup: {err:?}"); + } me.maybe_request_demo_cert(); Ok(me) } @@ -170,21 +295,244 @@ impl AppState { &self.inner.config } - fn quote_response(&self, report_data: [u8; 64]) -> Result<GetQuoteResponse> { + pub(crate) fn health(&self) -> Option<&crate::health::HealthMonitor> { + self.inner.health.as_deref() + } + + pub(crate) fn quote_response(&self, report_data: [u8; 64]) -> Result<GetQuoteResponse> { self.inner .platform .quote_response(report_data, &self.inner.vm_config) } - fn attest_response(&self, report_data: [u8; 64]) -> Result<AttestResponse> { - self.inner.platform.attest_response(report_data) + /// Encoded attestation bytes for `Attest`, in the wire form the calling + /// surface commits to. + pub(crate) fn attest_cvm( + &self, + report_data: [u8; 64], + wire: AttestationWire, + ) -> Result<Vec<u8>> { + wire.apply(self.inner.platform.attest_cvm(report_data)?) + .to_bytes() + } + + /// The application's root secp256k1 key, the root of every derived key and + /// the signer of the first link of every signature chain. + pub(crate) fn app_root_k256_key(&self) -> &[u8] { + &self.inner.keys.k256_key + } + + /// The same key, parsed. Shared rather than re-parsed per request. + pub(crate) fn app_root_signing_key(&self) -> Result<&SigningKey> { + let Some(key) = self.inner.app_root_signing_key.as_ref() else { + anyhow::bail!("the app root k256 key is not a valid secp256k1 scalar"); + }; + Ok(key) + } + + /// The KMS root key's signature over the app root public key: the second + /// link of every signature chain, produced outside this agent and passed + /// through byte-for-byte on both API surfaces. + pub(crate) fn kms_k256_signature(&self) -> &[u8] { + &self.inner.keys.k256_signature + } + + pub(crate) fn cloud_vendor(&self) -> &str { + &self.inner.cloud_vendor + } + + pub(crate) fn cloud_product(&self) -> &str { + &self.inner.cloud_product + } + + /// The decoded identity, computed at most once on success. + /// + /// Populated at construction; the decode below only runs when that attempt + /// failed, so a platform that could not attest at boot still answers later + /// rather than staying broken for the life of the process. It runs at most + /// once per [`IDENTITY_RETRY_INTERVAL`], because the callers reaching it + /// include anonymous ones. Within the window the caller is told the + /// identity is unavailable and when the next attempt is, and the platform + /// is not touched at all. + pub(crate) fn identity(&self) -> Result<Arc<AppIdentity>> { + if let Some(identity) = self + .inner + .identity + .read() + .or_panic("lock should never fail") + .as_ref() + { + return Ok(identity.clone()); + } + // Two callers arriving together on a cold cache may both attempt once. + // That is the same race the cache has always had, and one extra quote + // is not worth holding a lock across the decode for. + if let Some(failed_at) = *self + .inner + .identity_last_failure + .lock() + .or_panic("lock should never fail") + { + let elapsed = failed_at.elapsed(); + if elapsed < IDENTITY_RETRY_INTERVAL { + let retry_in = (IDENTITY_RETRY_INTERVAL - elapsed).as_secs() + 1; + anyhow::bail!( + "the app identity is unavailable: decoding it failed and the next attempt is at most {retry_in}s away" + ); + } + } + // Blocking, and deliberately left on the executor: the throttle above + // caps this at one quote per interval for the whole process, which is + // far short of what would justify a `spawn_blocking` hop and making + // every caller of `identity()` async to reach it. + let identity = match decode_identity(&self.inner) { + Ok(identity) => Arc::new(identity), + Err(err) => { + *self + .inner + .identity_last_failure + .lock() + .or_panic("lock should never fail") = Some(Instant::now()); + return Err(err); + } + }; + *self + .inner + .identity + .write() + .or_panic("lock should never fail") = Some(identity.clone()); + Ok(identity) + } + + /// The VM's hardware configuration, as the VMM produced it. + pub(crate) fn vm_config(&self) -> &str { + &self.inner.vm_config + } + + pub(crate) fn gpu_attestor(&self) -> &crate::gpu_attest::GpuAttestor { + &self.inner.gpu_attestor + } + + pub(crate) async fn issue_cert( + &self, + key: &KeyPair, + config: CertConfigV2, + wire: AttestationWire, + ) -> Result<Vec<String>> { + self.inner.issue_cert(key, config, wire).await + } +} + +/// Generate the fresh P-256 key that backs a certificate the agent issues. +/// +/// Random, not derived: the certificate is minted per call, so there is +/// nothing for a stable key to buy, and a key nobody can re-derive is a +/// smaller thing to hold. +/// +/// The `Failed to ...` contexts are the v0.5.11 strings verbatim. They are what +/// a frozen-surface caller sees, so they keep their original capitalisation +/// rather than following the lowercase house rule. +fn generate_cert_key() -> Result<KeyPair> { + let mut seed = [0u8; 32]; + SystemRandom::new() + .fill(&mut seed) + .context("Failed to generate secure seed")?; + derive_p256_key_pair_from_bytes(&seed, &[]).context("Failed to derive key") +} + +/// The certificate request fields both surfaces take. +/// +/// The two wire messages are different types with identical fields, so this is +/// where they meet. Without it the shared body is copied per surface and the +/// copies drift. +pub(crate) struct CertRequestFields { + pub(crate) subject: String, + pub(crate) alt_names: Vec<String>, + pub(crate) usage_ra_tls: bool, + pub(crate) usage_server_auth: bool, + pub(crate) usage_client_auth: bool, + pub(crate) with_app_info: bool, + pub(crate) not_before: Option<u64>, + pub(crate) not_after: Option<u64>, +} + +/// The wire form of an attestation the agent hands out, either directly +/// (`Attest`) or embedded in a CSR and the certificate issued from it +/// (`GetTlsKey`, `IssueCert`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum AttestationWire { + /// The v0.5.11 behaviour, for the frozen surfaces: legacy SCALE whenever + /// the attestation can be represented by it, MessagePack otherwise. + /// + /// Relying parties before 0.5.9 decode only SCALE. That includes a KMS + /// before 0.5.9 signing the CSR, and a source KMS before 0.5.9 verifying + /// the certificate a KMS onboards with. + Legacy, + /// Always the MessagePack V1 schema, for `dstack.guest.v1`. v1 has no + /// released client to stay compatible with, so it commits to one wire + /// format instead of letting the attestation's contents pick it. + MsgpackV1, +} + +impl AttestationWire { + pub(crate) fn apply(self, attestation: VersionedAttestation) -> VersionedAttestation { + match self { + Self::Legacy => attestation, + Self::MsgpackV1 => VersionedAttestation::V1 { + attestation: attestation.into_v1(), + }, + } } } +/// A freshly issued certificate and the key that backs it. +pub(crate) struct IssuedCert { + pub(crate) key: String, + pub(crate) certificate_chain: Vec<String>, +} + +/// Validate, generate a key, build the CSR, and get it signed. +/// +/// The whole body of the frozen `GetTlsKey` and of v1's `IssueCert`: they +/// differ only in which message type carries the fields in and the result out. +pub(crate) async fn issue_cert_for_request( + state: &AppState, + request: CertRequestFields, + wire: AttestationWire, +) -> Result<IssuedCert> { + validate_cert_validity(request.not_before, request.not_after)?; + let key = generate_cert_key()?; + let config = CertConfigV2 { + org_name: None, + subject: request.subject, + subject_alt_names: request.alt_names, + usage_server_auth: request.usage_server_auth, + usage_client_auth: request.usage_client_auth, + ext_quote: request.usage_ra_tls, + ext_app_info: request.with_app_info, + not_after: request.not_after, + not_before: request.not_before, + }; + let certificate_chain = state.issue_cert(&key, config, wire).await?; + Ok(IssuedCert { + key: key.serialize_pem(), + certificate_chain, + }) +} + pub struct InternalRpcHandler { state: AppState, } +impl InternalRpcHandler { + /// Only the router constructs this in a running agent; the v1 tests build + /// one directly to assert the unversioned surface still answers as it did. + #[cfg(test)] + pub(crate) fn new(state: AppState) -> Self { + Self { state } + } +} + pub async fn get_info(state: &AppState, external: bool) -> Result<AppInfo> { let hide_tcb_info = external && !state.config().app_compose.public_tcbinfo; let versioned_attestation = state.inner.info_attestation()?; @@ -250,29 +598,38 @@ pub async fn get_info(state: &AppState, external: bool) -> Result<AppInfo> { }) } +pub(crate) fn validate_cert_validity( + not_before: Option<u64>, + not_after: Option<u64>, +) -> Result<()> { + if let (Some(not_before), Some(not_after)) = (not_before, not_after) { + if not_before >= not_after { + anyhow::bail!("not_before must be earlier than not_after"); + } + } + Ok(()) +} + impl DstackGuestRpc for InternalRpcHandler { async fn get_tls_key(self, request: GetTlsKeyArgs) -> anyhow::Result<GetTlsKeyResponse> { - let mut seed = [0u8; 32]; - SystemRandom::new() - .fill(&mut seed) - .context("Failed to generate secure seed")?; - let derived_key = - derive_p256_key_pair_from_bytes(&seed, &[]).context("Failed to derive key")?; - let config = CertConfigV2 { - org_name: None, - subject: request.subject, - subject_alt_names: request.alt_names, - usage_server_auth: request.usage_server_auth, - usage_client_auth: request.usage_client_auth, - ext_quote: request.usage_ra_tls, - ext_app_info: request.with_app_info, - not_after: request.not_after, - not_before: request.not_before, - }; - let certificate_chain = self.state.inner.issue_cert(&derived_key, config).await?; + let issued = issue_cert_for_request( + &self.state, + CertRequestFields { + subject: request.subject, + alt_names: request.alt_names, + usage_ra_tls: request.usage_ra_tls, + usage_server_auth: request.usage_server_auth, + usage_client_auth: request.usage_client_auth, + with_app_info: request.with_app_info, + not_before: request.not_before, + not_after: request.not_after, + }, + AttestationWire::Legacy, + ) + .await?; Ok(GetTlsKeyResponse { - key: derived_key.serialize_pem(), - certificate_chain, + key: issued.key, + certificate_chain: issued.certificate_chain, }) } @@ -308,10 +665,10 @@ impl DstackGuestRpc for InternalRpcHandler { let msg_to_sign = format!("{}:{}", request.purpose, pubkey_hex); let app_signing_key = SigningKey::from_slice(k256_app_key).context("Failed to parse app k256 key")?; - let digest = Keccak256::new_with_prefix(msg_to_sign); - let (signature, recid) = app_signing_key.sign_digest_recoverable(digest)?; - let mut signature = signature.to_vec(); - signature.push(recid.to_byte()); + // The shared 65-byte `r || s || v` envelope. Byte-identical to the copy + // this replaced -- `get_key_pins_the_frozen_chain_link` is the vector + // that says so, and it exists for exactly this de-duplication. + let signature = sign_recoverable_keccak256(&app_signing_key, msg_to_sign.as_bytes())?; Ok(GetKeyResponse { key, @@ -324,6 +681,16 @@ impl DstackGuestRpc for InternalRpcHandler { self.state.quote_response(report_data) } + /// Always fails. See the RPC's doc comment in agent_rpc.proto: the method + /// exists so a pre-0.6 client learns why its events stopped being recorded, + /// instead of the bare `Service not found` a deleted method would answer + /// with, which says nothing about the removal. + async fn emit_event(self, _request: EmitEventArgs) -> Result<()> { + anyhow::bail!( + "EmitEvent was removed in dstack 0.6.0; runtime RTMR3 events are system-owned and cannot be extended by apps" + ) + } + async fn info(self) -> Result<AppInfo> { get_info(&self.state, false).await } @@ -387,6 +754,18 @@ impl DstackGuestRpc for InternalRpcHandler { }) } + /// Deprecated, kept for 0.5.x clients only. See the RPC's doc comment in + /// agent_rpc.proto. + /// + /// k256 rejects a non-canonical (high-S) signature, so a malleated copy of a + /// valid signature does not verify. Keep it that way: 0.5.x answered the + /// same, and callers may be treating this answer as a uniqueness check. + /// + /// The rejection is in the verification, not in the parsing -- `from_slice` + /// only rejects an `r` or `s` outside `1..n`, and a malleated `s` is still + /// in range. So a caller sees HTTP 200 with `valid: false`, not the 400 a + /// parse failure would produce. Same security answer, different status + /// code, and the status code is the part a client branches on. async fn verify(self, request: VerifyRequest) -> Result<VerifyResponse> { let algorithm = normalize_algorithm(&request.algorithm); let valid = match algorithm { @@ -423,7 +802,11 @@ impl DstackGuestRpc for InternalRpcHandler { async fn attest(self, request: RawQuoteArgs) -> Result<AttestResponse> { let report_data = pad64(&request.report_data).context("Report data is too long")?; - self.state.attest_response(report_data) + Ok(AttestResponse { + attestation: self + .state + .attest_cvm(report_data, AttestationWire::Legacy)?, + }) } async fn version(self) -> Result<WorkerVersion> { @@ -443,7 +826,7 @@ fn normalize_algorithm(algorithm: &str) -> &str { } } -fn pad64(data: &[u8]) -> Option<[u8; 64]> { +pub(crate) fn pad64(data: &[u8]) -> Option<[u8; 64]> { if data.len() > 64 { return None; } @@ -490,7 +873,11 @@ impl TappdRpc for InternalRpcHandlerV0 { not_before: None, not_after: None, }; - let certificate_chain = self.state.inner.issue_cert(&derived_key, config).await?; + let certificate_chain = self + .state + .inner + .issue_cert(&derived_key, config, AttestationWire::Legacy) + .await?; Ok(GetTlsKeyResponse { key: derived_key.serialize_pem(), certificate_chain, @@ -513,16 +900,16 @@ impl TappdRpc for InternalRpcHandlerV0 { } else { &request.hash_algorithm }; - let prefix = if hash_algorithm == "raw" { - "".into() - } else { - QuoteContentType::AppData.tag().to_string() - }; let content_type = if request.prefix.is_empty() { QuoteContentType::AppData } else { QuoteContentType::Custom(&request.prefix) }; + let prefix = if hash_algorithm == "raw" { + "".into() + } else { + content_type.tag().to_string() + }; let report_data = content_type.to_report_data_with_hash(&request.report_data, &request.hash_algorithm)?; let response = self.state.quote_response(report_data)?; @@ -587,54 +974,87 @@ impl WorkerRpc for ExternalRpcHandler { }) } + /// Legacy and frozen at the v0.5.11 shape. See the RPC's doc comment in + /// agent_rpc.proto. + /// + /// Returns a `GetQuoteResponse`, which only Intel TDX can fill, so this + /// fails on every other platform exactly as `GetQuote` does. v1 ships no + /// counterpart that lifts the limitation: a v1 application attests its own + /// key instead -- derive it at `/v1/GetKey`, commit the public key into + /// `report_data`, call `/v1/Attest`, and serve the result to relying + /// parties. See the `Worker` service comment in agent_rpc_v1.proto. + /// + /// The report data comes from `app_key_report_data`, which commits to the + /// key `Sign` derives -- same path, purpose and base algorithm -- so the + /// attested public key is the one that actually signs. async fn get_attestation_for_app_key( self, request: GetAttestationForAppKeyRequest, ) -> Result<GetQuoteResponse> { + let report_data = self.app_key_report_data(&request.algorithm).await?; + self.state.quote_response(report_data) + } +} + +impl ExternalRpcHandler { + /// Derive the app key for `algorithm` and build the DIP-1 report data that + /// commits to its public key. + /// + /// The caller cannot compute this itself -- it does not know the public key + /// until the key is derived here -- which is why attesting an app key needs + /// its own method instead of the caller-supplied report data `GetQuote` + /// and `Attest` take. + pub(crate) async fn app_key_report_data(&self, algorithm: &str) -> Result<[u8; 64]> { + let algorithm = normalize_algorithm(algorithm); + // Prehashing is a signing mode, not a key type: the same secp256k1 key + // signs both ways, so derive it under the base name. `Sign` does the + // same, and without this the prehashed name reaches `get_key` verbatim + // and comes back "Unsupported algorithm". + let key_algorithm = match algorithm { + "secp256k1_prehashed" => "secp256k1", + other => other, + }; let key_response = InternalRpcHandler { state: self.state.clone(), } .get_key(GetKeyArgs { path: "vms".to_string(), purpose: "signing".to_string(), - algorithm: request.algorithm.clone(), + algorithm: key_algorithm.to_string(), }) .await?; - let algorithm = normalize_algorithm(&request.algorithm); - match algorithm { + let (prefix, pubkey) = match algorithm { "ed25519" => { let key_bytes: [u8; 32] = key_response .key .try_into() .ok() .context("Key is incorrect")?; - let ed25519_key = Ed25519SigningKey::from_bytes(&key_bytes); - let ed25519_pubkey = ed25519_key.verifying_key().to_bytes(); - - let mut ed25519_report_data = [0u8; 64]; - let ed25519_b64 = URL_SAFE_NO_PAD.encode(ed25519_pubkey); - let ed25519_report_string = format!("dip1::ed25519-pk:{}", ed25519_b64); - let ed_bytes = ed25519_report_string.as_bytes(); - ed25519_report_data[..ed_bytes.len()].copy_from_slice(ed_bytes); - - self.state.quote_response(ed25519_report_data) + let key = Ed25519SigningKey::from_bytes(&key_bytes); + ("dip1::ed25519-pk:", key.verifying_key().to_bytes().to_vec()) } "secp256k1" | "secp256k1_prehashed" => { - let secp256k1_key = SigningKey::from_slice(&key_response.key) + let key = SigningKey::from_slice(&key_response.key) .context("Failed to parse secp256k1 key")?; - let secp256k1_pubkey = secp256k1_key.verifying_key().to_sec1_bytes(); - - let mut secp256k1_report_data = [0u8; 64]; - let secp256k1_b64 = URL_SAFE_NO_PAD.encode(secp256k1_pubkey); - let secp256k1_report_string = format!("dip1::secp256k1c-pk:{}", secp256k1_b64); - let secp_bytes = secp256k1_report_string.as_bytes(); - secp256k1_report_data[..secp_bytes.len()].copy_from_slice(secp_bytes); - - self.state.quote_response(secp256k1_report_data) + ( + "dip1::secp256k1c-pk:", + key.verifying_key().to_sec1_bytes().to_vec(), + ) } - _ => Err(anyhow::anyhow!("Unsupported algorithm")), + _ => return Err(anyhow::anyhow!("Unsupported algorithm")), + }; + + let report_string = format!("{prefix}{}", URL_SAFE_NO_PAD.encode(pubkey)); + let bytes = report_string.as_bytes(); + let mut report_data = [0u8; 64]; + // A longer public key encoding than the 64 bytes of report data would + // otherwise truncate into a valid-looking commitment to a different key. + if bytes.len() > report_data.len() { + anyhow::bail!("report data for {algorithm} does not fit in 64 bytes"); } + report_data[..bytes.len()].copy_from_slice(bytes); + Ok(report_data) } } @@ -649,24 +1069,29 @@ impl RpcCall<AppState> for ExternalRpcHandler { } #[cfg(test)] -mod tests { +// `pub(crate)` so the v1 handler's tests can build a state from the same +// fixture. Two fixtures would let the two surfaces be tested against different +// app root keys, which is exactly what the cross-version key assertions check. +pub(crate) mod tests { use super::*; use crate::{ backend::PlatformBackend, config::{AppComposeWrapper, Config}, }; - use dstack_guest_agent_rpc::{GetAttestationForAppKeyRequest, SignRequest}; - use dstack_types::{AppCompose, AppKeys, KeyProvider}; + use dstack_attest::attestation::AttestationVerifier; + use dstack_guest_agent_rpc::v0::{GetAttestationForAppKeyRequest, SignRequest}; + use dstack_types::{AppCompose, AppKeys, EventLogVersion, KeyProvider}; use ed25519_dalek::ed25519::signature::hazmat::PrehashVerifier; use ed25519_dalek::{ Signature as Ed25519Signature, Verifier, VerifyingKey as Ed25519VerifyingKey, }; use k256::ecdsa::{Signature as K256Signature, VerifyingKey}; - use ra_tls::attestation::{AttestationV1, VersionedAttestation}; - use sha2::Sha256; + use ra_tls::attestation::{AttestationV1, PlatformEvidence, VersionedAttestation}; + use sha2::{Digest as _, Sha256}; use std::collections::HashSet; use std::convert::TryFrom; use std::io::Write; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; fn extract_pubkey_from_report_data(report_data: &[u8], prefix: &str) -> Result<Vec<u8>> { let end = report_data @@ -684,7 +1109,89 @@ mod tests { } } - async fn setup_test_state() -> (AppState, tempfile::NamedTempFile) { + /// The attestation embedded in a PEM certificate the agent issued, in the + /// wire form it was embedded with. + pub(crate) fn embedded_attestation(pem: &str) -> VersionedAttestation { + use base64::engine::general_purpose::STANDARD; + let body: String = pem + .lines() + .filter(|line| !line.starts_with("-----")) + .collect(); + let der = STANDARD.decode(body).expect("certificate PEM body"); + ra_tls::attestation::from_der(&der) + .expect("decode certificate attestation") + .expect("certificate carries an attestation") + } + + pub(crate) async fn setup_test_state() -> (AppState, tempfile::NamedTempFile) { + setup_test_state_with_platform(None).await + } + + /// How many times the fixture platform was asked to attest for `Info`, and + /// whether it should refuse. Counting the calls is the only way to see the + /// identity throttle work: what it changes is how often the platform is + /// touched, not what any single call returns. + #[derive(Default)] + struct InfoAttestationProbe { + calls: AtomicUsize, + failing: AtomicBool, + } + + impl InfoAttestationProbe { + fn failing() -> Self { + Self { + calls: AtomicUsize::new(0), + failing: AtomicBool::new(true), + } + } + + fn calls(&self) -> usize { + self.calls.load(Ordering::Relaxed) + } + + fn set_failing(&self, failing: bool) { + self.failing.store(failing, Ordering::Relaxed); + } + } + + /// The fixture state, built against a platform the probe watches. + async fn setup_test_state_with_probe( + probe: Arc<InfoAttestationProbe>, + ) -> (AppState, tempfile::NamedTempFile) { + build_test_state(None, |_| {}, probe).await + } + + /// The same state with the app having opted into publishing its TCB info, + /// which is what unlocks the document fields on the external surface. + pub(crate) async fn setup_test_state_with_public_tcbinfo() -> (AppState, tempfile::NamedTempFile) + { + build_test_state( + None, + |config| { + config.app_compose.app_compose.public_tcbinfo = true; + config.app_compose.raw = r#"{"name":"test"}"#.to_string(); + }, + Arc::default(), + ) + .await + } + + /// The same state, with the fixture's platform evidence swapped out. + /// `None` keeps the fixture's Intel TDX evidence. + pub(crate) async fn setup_test_state_with_platform( + platform: Option<PlatformEvidence>, + ) -> (AppState, tempfile::NamedTempFile) { + build_test_state(platform, |_| {}, Arc::default()).await + } + + /// The one fixture body. `configure` adjusts the app config before the + /// state is built, which is cheaper and clearer than rebuilding an + /// already-shared `Arc` afterwards. + async fn build_test_state( + platform: Option<PlatformEvidence>, + configure: impl FnOnce(&mut Config), + probe: Arc<InfoAttestationProbe>, + ) -> (AppState, tempfile::NamedTempFile) { let mut temp_attestation_file = tempfile::NamedTempFile::new().unwrap(); let attestation = include_bytes!("../fixtures/attestation.bin"); @@ -696,7 +1203,9 @@ mod tests { name: String::new(), features: Vec::new(), runner: String::new(), + snapshotter: None, docker_compose_file: None, + init_script: Vec::new(), public_logs: false, public_sysinfo: false, public_tcbinfo: false, @@ -709,9 +1218,12 @@ mod tests { no_instance_id: false, secure_time: false, storage_fs: None, + storage_discard: true, swap_size: 0, + event_log_version: EventLogVersion::V1, port_policy: Default::default(), requirements: None, + verity_volumes: Vec::new(), }; let dummy_appcompose_wrapper = AppComposeWrapper { @@ -719,13 +1231,13 @@ mod tests { raw: String::new(), }; - let dummy_config = Config { + let mut dummy_config = Config { keys_file: String::new(), app_compose: dummy_appcompose_wrapper, sys_config_file: String::new().into(), - pccs_url: None, data_disks: HashSet::new(), }; + configure(&mut dummy_config); const DUMMY_PEM_KEY: &str = r#"-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCSeV81CKVqILf/ @@ -796,12 +1308,14 @@ pNs85uhOZE8z2jr8Pg== }, }; - let dummy_cert_client = CertRequestClient::create(&dummy_keys, None, String::new()) + let verifier = Arc::new(AttestationVerifier::new_prod(None).unwrap()); + let dummy_cert_client = CertRequestClient::create(&dummy_keys, verifier, String::new()) .await .expect("Failed to create CertRequestClient"); struct TestSimulatorPlatform { attestation: VersionedAttestation, + probe: Arc<InfoAttestationProbe>, } fn patch_report_data( @@ -813,6 +1327,10 @@ pNs85uhOZE8z2jr8Pg== impl PlatformBackend for TestSimulatorPlatform { fn attestation_for_info(&self) -> Result<VersionedAttestation> { + self.probe.calls.fetch_add(1, Ordering::Relaxed); + if self.probe.failing.load(Ordering::Relaxed) { + anyhow::bail!("the platform cannot attest right now"); + } Ok(self.attestation.clone()) } @@ -820,6 +1338,10 @@ pNs85uhOZE8z2jr8Pg== let report_data = ra_tls::attestation::QuoteContentType::RaTlsCert.to_report_data(pubkey); let attestation = patch_report_data(&self.attestation, report_data); + // Same legacy mirroring as `attest_cvm` below. + if matches!(self.attestation, VersionedAttestation::V0 { .. }) { + return Ok(attestation.try_into_legacy()?.into_versioned()); + } Ok(VersionedAttestation::V1 { attestation }) } @@ -830,7 +1352,9 @@ pNs85uhOZE8z2jr8Pg== ) -> Result<GetQuoteResponse> { let attestation = patch_report_data(&self.attestation, report_data); let Some(quote) = attestation.platform.tdx_quote().map(ToOwned::to_owned) else { - return Err(anyhow::anyhow!("Quote not found")); + return Err(anyhow::anyhow!( + "GetQuote is Intel TDX only, use Attest on this platform" + )); }; Ok(GetQuoteResponse { quote, @@ -840,15 +1364,18 @@ pNs85uhOZE8z2jr8Pg== .unwrap_or_default(), report_data: report_data.to_vec(), vm_config: vm_config.to_string(), - attestation: Vec::new(), }) } - fn attest_response(&self, report_data: [u8; 64]) -> Result<AttestResponse> { + fn attest_cvm(&self, report_data: [u8; 64]) -> Result<VersionedAttestation> { let attestation = patch_report_data(&self.attestation, report_data); - Ok(AttestResponse { - attestation: VersionedAttestation::V1 { attestation }.to_bytes()?, - }) + // Mirror `RealPlatform`: a dstack TDX attestation whose runtime + // events are all V1 comes out in the legacy form, which is the + // case the v1 handler has to re-encode. + if matches!(self.attestation, VersionedAttestation::V0 { .. }) { + return Ok(attestation.try_into_legacy()?.into_versioned()); + } + Ok(VersionedAttestation::V1 { attestation }) } } @@ -859,11 +1386,36 @@ pNs85uhOZE8z2jr8Pg== cert_client: dummy_cert_client, demo_cert: RwLock::new(String::new()), platform: Arc::new(TestSimulatorPlatform { - attestation: VersionedAttestation::from_bytes( - &std::fs::read(temp_attestation_file.path()).unwrap(), - ) - .unwrap(), + attestation: { + let fixture = VersionedAttestation::from_bytes( + &std::fs::read(temp_attestation_file.path()).unwrap(), + ) + .unwrap(); + match platform { + None => fixture, + Some(evidence) => { + let mut attestation = fixture.into_v1(); + attestation.platform = evidence; + VersionedAttestation::V1 { attestation } + } + } + }, + probe, }), + health: None, + // Pinned to a path that cannot exist, so no test ever spawns a + // real collection against a host GPU and every test sees the + // same "this image cannot attest a GPU" answer. + gpu_attestor: crate::gpu_attest::GpuAttestor::with_nvattest_path( + "/nonexistent/nvattest", + ), + app_root_signing_key: SigningKey::from_slice(&DUMMY_K256_KEY).ok(), + identity: RwLock::new(None), + identity_last_failure: Mutex::new(None), + // Read the same way production does, so a test comparing v1 `Info` + // against v0 `get_info` compares like with like. + cloud_vendor: read_dmi_file("sys_vendor"), + cloud_product: read_dmi_file("product_name"), }; ( @@ -874,60 +1426,6 @@ pNs85uhOZE8z2jr8Pg== ) } - #[tokio::test] - async fn test_verify_ed25519_success() { - let (state, _guard) = setup_test_state().await; - let handler = InternalRpcHandler { - state: state.clone(), - }; - let data_to_sign = b"test message for ed25519"; - let sign_request = SignRequest { - algorithm: "ed25519".to_string(), - data: data_to_sign.to_vec(), - }; - - let sign_response = handler.sign(sign_request).await.unwrap(); - - let verify_request = VerifyRequest { - algorithm: "ed25519".to_string(), - data: data_to_sign.to_vec(), - signature: sign_response.signature, - public_key: sign_response.public_key, - }; - let handler = InternalRpcHandler { - state: state.clone(), - }; - let verify_response = handler.verify(verify_request).await.unwrap(); - assert!(verify_response.valid); - } - - #[tokio::test] - async fn test_verify_secp256k1_success() { - let (state, _guard) = setup_test_state().await; - let handler = InternalRpcHandler { - state: state.clone(), - }; - let data_to_sign = b"test message for secp256k1"; - let sign_request = SignRequest { - algorithm: "secp256k1".to_string(), - data: data_to_sign.to_vec(), - }; - - let sign_response = handler.sign(sign_request).await.unwrap(); - - let verify_request = VerifyRequest { - algorithm: "secp256k1".to_string(), - data: data_to_sign.to_vec(), - signature: sign_response.signature, - public_key: sign_response.public_key, - }; - let handler = InternalRpcHandler { - state: state.clone(), - }; - let verify_response = handler.verify(verify_request).await.unwrap(); - assert!(verify_response.valid); - } - #[tokio::test] async fn test_sign_ed25519_success() { let (state, _guard) = setup_test_state().await; @@ -942,16 +1440,11 @@ pNs85uhOZE8z2jr8Pg== let response = handler.sign(request).await.unwrap(); - let attestation_response = ExternalRpcHandler::new(state) - .get_attestation_for_app_key(GetAttestationForAppKeyRequest { - algorithm: "ed25519".to_string(), - }) + let report_data = ExternalRpcHandler::new(state) + .app_key_report_data("ed25519") .await .unwrap(); - - let pk_bytes = - extract_pubkey_from_report_data(&attestation_response.report_data, "dip1::ed25519-pk:") - .unwrap(); + let pk_bytes = extract_pubkey_from_report_data(&report_data, "dip1::ed25519-pk:").unwrap(); let public_key = Ed25519VerifyingKey::try_from(pk_bytes.as_slice()).unwrap(); let signature = Ed25519Signature::try_from(response.signature.as_slice()).unwrap(); @@ -972,18 +1465,12 @@ pNs85uhOZE8z2jr8Pg== let response = handler.sign(request).await.unwrap(); - let attestation_response = ExternalRpcHandler::new(state) - .get_attestation_for_app_key(GetAttestationForAppKeyRequest { - algorithm: "secp256k1".to_string(), - }) + let report_data = ExternalRpcHandler::new(state) + .app_key_report_data("secp256k1") .await .unwrap(); - - let pk_bytes = extract_pubkey_from_report_data( - &attestation_response.report_data, - "dip1::secp256k1c-pk:", - ) - .unwrap(); + let pk_bytes = + extract_pubkey_from_report_data(&report_data, "dip1::secp256k1c-pk:").unwrap(); let public_key = VerifyingKey::from_sec1_bytes(&pk_bytes).unwrap(); let signature = K256Signature::try_from(response.signature.as_slice()).unwrap(); @@ -1007,18 +1494,12 @@ pNs85uhOZE8z2jr8Pg== let response = handler.sign(request).await.unwrap(); - let attestation_response = ExternalRpcHandler::new(state) - .get_attestation_for_app_key(GetAttestationForAppKeyRequest { - algorithm: "secp256k1".to_string(), - }) + let report_data = ExternalRpcHandler::new(state) + .app_key_report_data("secp256k1") .await .unwrap(); - - let pk_bytes = extract_pubkey_from_report_data( - &attestation_response.report_data, - "dip1::secp256k1c-pk:", - ) - .unwrap(); + let pk_bytes = + extract_pubkey_from_report_data(&report_data, "dip1::secp256k1c-pk:").unwrap(); let public_key = VerifyingKey::from_sec1_bytes(&pk_bytes).unwrap(); let signature = K256Signature::try_from(response.signature.as_slice()).unwrap(); @@ -1064,49 +1545,150 @@ pNs85uhOZE8z2jr8Pg== assert_eq!(result.unwrap_err().to_string(), "Unsupported algorithm"); } + const ED25519_REPORT_DATA: &str = + "dip1::ed25519-pk:5Pbre1Amf1hrp2V2bbfKlIfxpQb2pJAmrgmhxgVoG9s\0\0\0\0"; + const SECP256K1_REPORT_DATA: &str = + "dip1::secp256k1c-pk:A6t_JdVkVdMAocH3f1f20WGT6JzdntxcXimUtEax8zc9"; + + /// The DIP-1 report data the frozen `Worker.GetAttestationForAppKey` + /// commits to, wrapped in a TDX quote. Pinned so the commitment format a + /// relying party parses cannot drift; v1 has no counterpart method, so + /// this is the only surface these bytes appear on. #[tokio::test] - async fn test_get_attestation_for_app_key_ed25519_success() { + async fn app_key_report_data_matches_its_vectors() { let (state, _guard) = setup_test_state().await; - let handler = ExternalRpcHandler::new(state.clone()); - let request = GetAttestationForAppKeyRequest { - algorithm: "ed25519".to_string(), - }; - - let response = handler.get_attestation_for_app_key(request).await.unwrap(); + for (algorithm, expected) in [ + ("ed25519", ED25519_REPORT_DATA), + ("secp256k1", SECP256K1_REPORT_DATA), + ] { + let report_data = ExternalRpcHandler::new(state.clone()) + .app_key_report_data(algorithm) + .await + .unwrap(); + assert_eq!(expected.as_bytes(), report_data.as_slice(), "{algorithm}"); + } + } - const EXPECTED_REPORT_DATA: &str = - "dip1::ed25519-pk:5Pbre1Amf1hrp2V2bbfKlIfxpQb2pJAmrgmhxgVoG9s\0\0\0\0"; - assert_eq!(EXPECTED_REPORT_DATA.as_bytes(), response.report_data); - assert!(response.attestation.is_empty()); + /// Prehashing changes how a key signs, not which key it is, so it must + /// commit to the same public key as the plain name. + #[tokio::test] + async fn app_key_report_data_accepts_secp256k1_prehashed() { + let (state, _guard) = setup_test_state().await; + let prehashed = ExternalRpcHandler::new(state.clone()) + .app_key_report_data("secp256k1_prehashed") + .await + .expect("secp256k1_prehashed must be accepted"); + let plain = ExternalRpcHandler::new(state) + .app_key_report_data("secp256k1") + .await + .unwrap(); + assert_eq!(prehashed, plain); } #[tokio::test] - async fn test_get_attestation_for_app_key_secp256k1_success() { + async fn app_key_report_data_rejects_an_unsupported_algorithm() { let (state, _guard) = setup_test_state().await; - let handler = ExternalRpcHandler::new(state.clone()); - let request = GetAttestationForAppKeyRequest { - algorithm: "secp256k1".to_string(), - }; + let result = ExternalRpcHandler::new(state) + .app_key_report_data("ecdsa") + .await; + assert_eq!(result.unwrap_err().to_string(), "Unsupported algorithm"); + } - let response = handler.get_attestation_for_app_key(request).await.unwrap(); + /// The frozen method returns a `GetQuoteResponse`, which only Intel TDX can + /// fill. That limitation is why v1 replaced the method with the + /// attest-your-own-key flow rather than porting it, so it has to stay + /// observable here. + #[tokio::test] + async fn get_attestation_for_app_key_is_tdx_only() { + let (state, _guard) = setup_test_state_with_platform(Some(PlatformEvidence::SevSnp { + report: vec![0u8; 1184], + cert_chain: Vec::new(), + mr_config: String::new(), + })) + .await; - const EXPECTED_REPORT_DATA: &str = - "dip1::secp256k1c-pk:A6t_JdVkVdMAocH3f1f20WGT6JzdntxcXimUtEax8zc9"; - assert_eq!(EXPECTED_REPORT_DATA.as_bytes(), response.report_data); - assert!(response.attestation.is_empty()); + let err = ExternalRpcHandler::new(state) + .get_attestation_for_app_key(GetAttestationForAppKeyRequest { + algorithm: "ed25519".to_string(), + }) + .await + .expect_err("the frozen method cannot answer without a TDX quote"); + assert!( + err.to_string().contains("Intel TDX only"), + "unexpected error: {err}" + ); } + /// v0 `GetTlsKey` embeds the attestation in the legacy form it always + /// did, both in the CSR the KMS decodes and in the certificate it signs. #[tokio::test] - async fn test_get_attestation_for_app_key_unsupported_algorithm_fails() { + async fn v0_get_tls_key_keeps_the_legacy_wire_format() { let (state, _guard) = setup_test_state().await; - let handler = ExternalRpcHandler::new(state); - let request = GetAttestationForAppKeyRequest { - algorithm: "ecdsa".to_string(), // Unsupported algorithm - }; + let response = InternalRpcHandler::new(state) + .get_tls_key(GetTlsKeyArgs { + subject: "example".to_string(), + usage_ra_tls: true, + ..Default::default() + }) + .await + .unwrap(); + let attestation = embedded_attestation(&response.certificate_chain[0]); + assert!(matches!(attestation, VersionedAttestation::V0 { .. })); + } - let result = handler.get_attestation_for_app_key(request).await; - assert!(result.is_err()); - assert_eq!(result.unwrap_err().to_string(), "Unsupported algorithm"); + /// v0 `Attest` keeps the legacy SCALE form for an attestation it can + /// represent, so relying parties older than 0.5.9 and KMS onboarding from an + /// older source keep decoding it. The v1 counterpart lives in + /// `rpc_service_v1::tests::attest_always_returns_msgpack`. + #[tokio::test] + async fn v0_attest_keeps_the_legacy_wire_format() { + let (state, _guard) = setup_test_state().await; + let response = InternalRpcHandler::new(state) + .attest(RawQuoteArgs { + report_data: b"hello".to_vec(), + }) + .await + .unwrap(); + assert_eq!(response.attestation.first(), Some(&0x00)); + let attestation = VersionedAttestation::from_bytes(&response.attestation).unwrap(); + assert!(matches!(attestation, VersionedAttestation::V0 { .. })); + assert_eq!(&attestation.into_v1().report_data().unwrap()[..5], b"hello"); + } + + #[tokio::test] + async fn get_attestation_for_app_key_answers_on_tdx() { + let (state, _guard) = setup_test_state().await; + let response = ExternalRpcHandler::new(state) + .get_attestation_for_app_key(GetAttestationForAppKeyRequest { + algorithm: "ed25519".to_string(), + }) + .await + .unwrap(); + assert_eq!(ED25519_REPORT_DATA.as_bytes(), response.report_data); + assert!(!response.quote.is_empty()); + } + + /// The frozen v0 signature chain, pinned byte for byte. + /// + /// Added when the keccak256 -> recoverable-sign -> `r || s || v` envelope + /// was de-duplicated into `ra_tls::api_v1`: without a vector here, + /// nothing would have caught the shared helper disagreeing with the copy it + /// replaced. RFC 6979 makes the signature deterministic, so this is exact. + #[tokio::test] + async fn get_key_pins_the_frozen_chain_link() { + let (state, _guard) = setup_test_state().await; + let response = InternalRpcHandler::new(state) + .get_key(GetKeyArgs { + path: "test".to_string(), + purpose: "signing".to_string(), + algorithm: "secp256k1".to_string(), + }) + .await + .unwrap(); + assert_eq!( + hex::encode(&response.signature_chain[0]), + "c8a3dcf06c4e95bd78a5d7a1c8fcff171fc5848cfae804c6fc11bda4dc5d4062379995390843827444992c4c0e4bac70f0f878e01b9fc8b98cd7126fe5a3876b01" + ); } #[test] @@ -1206,6 +1788,19 @@ pNs85uhOZE8z2jr8Pg== assert_eq!(resp_default.key, resp_secp.key); } + #[test] + fn test_tls_certificate_validity_order() { + assert!(validate_cert_validity(None, None).is_ok()); + assert!(validate_cert_validity(Some(10), Some(11)).is_ok()); + assert_eq!( + validate_cert_validity(Some(11), Some(11)) + .unwrap_err() + .to_string(), + "not_before must be earlier than not_after" + ); + assert!(validate_cert_validity(Some(12), Some(11)).is_err()); + } + #[tokio::test] async fn test_get_key_unsupported_algorithm_fails() { let (state, _guard) = setup_test_state().await; @@ -1231,6 +1826,33 @@ pNs85uhOZE8z2jr8Pg== assert!(!response.version.is_empty()); } + #[tokio::test] + async fn test_tdx_quote_reports_effective_prefix() { + let (state, _guard) = setup_test_state().await; + + let default = InternalRpcHandlerV0 { + state: state.clone(), + } + .tdx_quote(TdxQuoteArgs { + report_data: b"test".to_vec(), + hash_algorithm: "sha512".to_string(), + prefix: "".to_string(), + }) + .await + .unwrap(); + assert_eq!(default.prefix, "app-data"); + + let custom = InternalRpcHandlerV0 { state } + .tdx_quote(TdxQuoteArgs { + report_data: b"test".to_vec(), + hash_algorithm: "sha512".to_string(), + prefix: "custom-domain".to_string(), + }) + .await + .unwrap(); + assert_eq!(custom.prefix, "custom-domain"); + } + #[tokio::test] async fn test_sign_k256_alias() { let (state, _guard) = setup_test_state().await; @@ -1258,4 +1880,200 @@ pNs85uhOZE8z2jr8Pg== // k256 alias should produce the same public key as secp256k1 assert_eq!(resp_k256.public_key, resp_secp.public_key); } + + /// Sign with `algorithm`, then verify the result through the legacy Verify + /// RPC -- the round trip a 0.5.x SDK performs. + async fn sign_then_verify( + algorithm: &str, + data: Vec<u8>, + ) -> (AppState, tempfile::NamedTempFile, SignResponse) { + let (state, guard) = setup_test_state().await; + let signed = InternalRpcHandler { + state: state.clone(), + } + .sign(SignRequest { + algorithm: algorithm.to_string(), + data: data.clone(), + }) + .await + .unwrap(); + + let verified = InternalRpcHandler { + state: state.clone(), + } + .verify(VerifyRequest { + algorithm: algorithm.to_string(), + data, + signature: signed.signature.clone(), + public_key: signed.public_key.clone(), + }) + .await + .unwrap(); + assert!(verified.valid); + + (state, guard, signed) + } + + #[tokio::test] + async fn verify_accepts_an_ed25519_signature_from_sign() { + sign_then_verify("ed25519", b"test message for ed25519".to_vec()).await; + } + + #[tokio::test] + async fn verify_accepts_a_secp256k1_signature_from_sign() { + sign_then_verify("secp256k1", b"test message for secp256k1".to_vec()).await; + } + + #[tokio::test] + async fn verify_accepts_a_secp256k1_prehashed_signature_from_sign() { + let digest = Sha256::digest(b"test message for secp256k1 prehashed"); + sign_then_verify("secp256k1_prehashed", digest.to_vec()).await; + } + + #[tokio::test] + async fn verify_rejects_tampered_data() { + let (state, _guard, signed) = + sign_then_verify("ed25519", b"original message".to_vec()).await; + + let response = InternalRpcHandler { state } + .verify(VerifyRequest { + algorithm: "ed25519".to_string(), + data: b"tampered message".to_vec(), + signature: signed.signature, + public_key: signed.public_key, + }) + .await + .unwrap(); + + assert!(!response.valid); + } + + /// A malleated (high-S) signature must not verify, and must come back as a + /// verdict rather than as an error. + /// + /// Both halves matter. k256 rejects high-S inside the verification, not in + /// `from_slice`, so the caller sees HTTP 200 with `valid: false` -- not the + /// 400 a parse failure would produce, and the status code is the part a + /// 0.5.x client branches on. A k256 upgrade that moved the check into + /// parsing would keep the security answer and silently change the status, + /// which is what `.expect()` below is here to catch. + #[tokio::test] + async fn verify_rejects_a_malleated_secp256k1_signature() { + let data = b"test message for secp256k1".to_vec(); + let (state, _guard, signed) = sign_then_verify("secp256k1", data.clone()).await; + + // `sign` emits low-S, so negating `s` yields the other encoding of the + // same signature -- the one an unnormalised verifier would also accept. + let signature = K256Signature::from_slice(&signed.signature).unwrap(); + let malleated = K256Signature::from_scalars(signature.r(), -signature.s()).unwrap(); + // `normalize_s` is `Some` only for a high-S signature, so this asserts + // the malleation is real and is exactly the twin of what just verified. + assert_eq!(malleated.normalize_s().as_ref(), Some(&signature)); + + let response = InternalRpcHandler { state } + .verify(VerifyRequest { + algorithm: "secp256k1".to_string(), + data, + signature: malleated.to_vec(), + public_key: signed.public_key, + }) + .await + .expect("a malleated signature is a verdict, not an error"); + + assert!(!response.valid); + } + + #[tokio::test] + async fn verify_unsupported_algorithm_fails() { + let (state, _guard) = setup_test_state().await; + let result = InternalRpcHandler { state } + .verify(VerifyRequest { + algorithm: "rsa".to_string(), + data: b"test message".to_vec(), + signature: vec![0; 64], + public_key: vec![0; 32], + }) + .await; + + assert_eq!(result.unwrap_err().to_string(), "Unsupported algorithm"); + } + + #[tokio::test] + async fn emit_event_reports_its_removal() { + let (state, _guard) = setup_test_state().await; + let result = InternalRpcHandler { state } + .emit_event(EmitEventArgs { + event: "test-event".to_string(), + payload: b"payload".to_vec(), + }) + .await; + + let err = result.unwrap_err().to_string(); + assert!(err.contains("removed in dstack 0.6.0"), "{err}"); + } + + /// A failed decode must be as cheap to repeat as a cached success is. + /// `identity()` is reached from the anonymous `/prpc/v1/Info`, and each + /// attempt is a hardware quote plus an RTMR replay under the global quote + /// lock -- retrying per call would give a caller on a broken platform the + /// very lever the cache exists to take away. + #[tokio::test] + async fn a_failed_identity_decode_is_not_retried_within_the_throttle_window() { + let probe = Arc::new(InfoAttestationProbe::failing()); + let (state, _guard) = setup_test_state_with_probe(probe.clone()).await; + + let err = state + .identity() + .expect_err("the platform refuses to attest"); + assert!(err.to_string().contains("cannot attest"), "{err}"); + assert_eq!(probe.calls(), 1); + + for _ in 0..8 { + let err = state.identity().expect_err("still throttled"); + assert!( + err.to_string().contains("the app identity is unavailable"), + "{err}" + ); + } + assert_eq!( + probe.calls(), + 1, + "the platform was asked again inside the throttle window" + ); + } + + /// The throttle bounds the retry rate; it must not turn a transient failure + /// into a permanent one. Once the window has passed the next call attests + /// again, and a success from then on is cached like any other. + #[tokio::test] + async fn the_identity_decode_is_retried_once_the_throttle_window_has_passed() { + let probe = Arc::new(InfoAttestationProbe::failing()); + let (state, _guard) = setup_test_state_with_probe(probe.clone()).await; + state + .identity() + .expect_err("the platform refuses to attest"); + + // Age the recorded failure rather than sleeping out the interval. + *state + .inner + .identity_last_failure + .lock() + .expect("lock should never fail") = Some( + Instant::now() + .checked_sub(IDENTITY_RETRY_INTERVAL) + .expect("the monotonic clock is older than the throttle window"), + ); + probe.set_failing(false); + + let identity = state.identity().expect("the platform recovered"); + assert_eq!(probe.calls(), 2); + + let again = state.identity().expect("a decoded identity is cached"); + assert!(Arc::ptr_eq(&identity, &again)); + assert_eq!( + probe.calls(), + 2, + "a success must be cached, not re-decoded per call" + ); + } } diff --git a/dstack/guest-agent/src/rpc_service_v1.rs b/dstack/guest-agent/src/rpc_service_v1.rs new file mode 100644 index 000000000..91a5141de --- /dev/null +++ b/dstack/guest-agent/src/rpc_service_v1.rs @@ -0,0 +1,1037 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! The `dstack.guest.v1` API surface. +//! +//! Two handlers, one per trust surface, mirroring the two unversioned ones: +//! +//! - [`V1RpcHandler`] on the internal socket at `/v1`, which hands out key +//! material because only the application itself can reach that socket; +//! - [`ExternalV1RpcHandler`] on the external listener at `/prpc/v1`, which +//! never does, because anyone who can route to the CVM can reach it. +//! +//! The unversioned handlers in [`crate::rpc_service`] keep serving `/` and +//! `/prpc` unchanged; a caller picks a surface by URL and nothing else. +//! +//! This handler shares every backend with the unversioned one -- the same +//! `AppState`, the same certificate client, the same platform attestation and +//! GPU attestor. What it does not share is key derivation and the +//! signature-chain claim, which are new in v1 and live in the `keys` submodule. +//! +//! v1 serves only what needs the TEE: deriving from the app root key, and +//! attesting. It has no `Sign` and no `Verify`, because a caller that can +//! reach this socket can get the private key from `GetKey` and do both +//! itself. + +use std::path::Path; + +use anyhow::{Context, Result}; +use dstack_guest_agent_rpc::v1::{ + dstack_guest_server::{DstackGuestRpc, DstackGuestServer}, + worker_server::{WorkerRpc, WorkerServer}, + AttestGpuRequest, AttestGpuResponse, AttestRequest, AttestResponse, GetKeyRequest, + GetKeyResponse, GpuEvidenceBundle, HealthRequest, HealthResponse, InfoRequest, InfoResponse, + IssueCertRequest, IssueCertResponse, VersionRequest, VersionResponse, +}; +use dstack_types::GPU_ATTESTATION_OUTPUT; +use fs_err as fs; +use ra_rpc::{CallContext, RpcCall}; +use tracing::warn; + +use crate::rpc_service::{issue_cert_for_request, pad64, AppState, AttestationWire}; + +pub(crate) mod keys; + +use keys::{Algorithm, AppKey}; + +/// The vendor every bundle this agent produces carries today. +const GPU_VENDOR: &str = "nvidia"; + +/// Format tag for evidence collected on demand, against a caller's nonce. +const GPU_FORMAT_ON_DEMAND: &str = "nvidia-nvattest-collect-evidence-json-v1"; + +/// Format tag for the record nvattest wrote at boot. +/// +/// Distinct from [`GPU_FORMAT_ON_DEMAND`] because the two answer different +/// questions -- "is this device genuine right now" versus "what did the boot +/// look like" -- and a verifier for one does not appraise the other. Sharing a +/// tag would leave a consumer no way to tell them apart. +const GPU_FORMAT_BOOTTIME: &str = "nvidia-nvattest-boottime-json-v1"; + +/// Read the GPU attestation output saved during boot. +/// +/// Returns the bytes exactly as they sit on disk. That exactness is the whole +/// contract: the only thing binding this evidence to the boot is sha256 over +/// precisely these bytes against the measured `gpu-attestation` event, so this +/// must never normalise, re-encode, or trim what it read. +/// +/// `None` when no output is available (no GPU attached, or attestation +/// disabled). +fn read_gpu_attestation(path: &Path) -> Option<Vec<u8>> { + match fs::read(path) { + Ok(attestation) => Some(attestation), + Err(err) => { + if err.kind() != std::io::ErrorKind::NotFound { + warn!("failed to read GPU attestation output: {err:?}"); + } + None + } + } +} + +/// GPU evidence to return alongside an attestation. +/// +/// Opt-in, so a caller that does not care about GPUs neither pays the disk read +/// nor carries the payload. Returns the same bundle shape `AttestGpu` uses, so +/// a consumer needs one parser rather than two; absence is the empty list. +fn boottime_gpu_evidence(include: bool, path: &Path) -> Vec<GpuEvidenceBundle> { + if !include { + return Vec::new(); + } + read_gpu_attestation(path) + .map(|evidence| { + vec![GpuEvidenceBundle { + vendor: GPU_VENDOR.to_string(), + format: GPU_FORMAT_BOOTTIME.to_string(), + evidence, + }] + }) + .unwrap_or_default() +} + +/// Build the v1 `Info` response. +/// +/// Reads identity from the cache `AppState` decoded once; see +/// [`crate::rpc_service::AppIdentity`] for why this must not attest per call. +/// +/// `hide_documents` applies the app's `public_tcbinfo` choice: with it set, the +/// three document fields come back empty and everything identifying the app +/// stays visible. Each hidden field is skipped rather than built and thrown +/// away, so the public listener does not clone an app-compose document per +/// anonymous call in order to discard it. +/// +/// This is close to, but not the same as, the line the frozen `Worker.Info` +/// draws. That one blanks `tcb_info` and `vm_config` and always serves +/// `key_provider_info`; v1 blanks `key_provider_info` too, because it names the +/// component that holds the app's keys and an external caller has no need for +/// it. The frozen behaviour is unchanged on its own surface. +fn info_response(state: &AppState, hide_documents: bool) -> Result<InfoResponse> { + let identity = state.identity()?; + let document = |value: &dyn Fn() -> String| { + if hide_documents { + String::new() + } else { + value() + } + }; + Ok(InfoResponse { + app_id: identity.app_id.clone(), + app_name: state.config().app_compose.name.clone(), + compose_hash: identity.compose_hash.clone(), + app_compose: document(&|| state.config().app_compose.raw.clone()), + instance_id: identity.instance_id.clone(), + device_id: identity.device_id.clone(), + os_image_hash: identity.os_image_hash.clone(), + mr_aggregated: identity.mr_aggregated.clone(), + vm_config: document(&|| state.vm_config().to_string()), + key_provider_info: document(&|| identity.key_provider_info.clone()), + cloud_vendor: state.cloud_vendor().to_string(), + cloud_product: state.cloud_product().to_string(), + }) +} + +/// The version both v1 services report. +fn version_response() -> VersionResponse { + VersionResponse { + version: crate::CARGO_PKG_VERSION.to_string(), + rev: crate::GIT_REV.to_string(), + } +} + +pub struct V1RpcHandler { + state: AppState, +} + +impl V1RpcHandler { + #[cfg(test)] + pub(crate) fn new(state: AppState) -> Self { + Self { state } + } + + /// Derive the key named by `(domain, algorithm)` together with its + /// signature chain. + fn derive(&self, domain: &str, algorithm: &str) -> Result<(AppKey, Vec<Vec<u8>>)> { + let algorithm = Algorithm::parse(algorithm)?; + let key = AppKey::derive(self.state.app_root_k256_key(), domain, algorithm)?; + let chain = vec![ + key.claim_signature(self.state.app_root_signing_key()?)?, + self.state.kms_k256_signature().to_vec(), + ]; + Ok((key, chain)) + } +} + +impl DstackGuestRpc for V1RpcHandler { + async fn issue_cert(self, request: IssueCertRequest) -> Result<IssueCertResponse> { + let issued = issue_cert_for_request( + &self.state, + crate::rpc_service::CertRequestFields { + subject: request.subject, + alt_names: request.alt_names, + usage_ra_tls: request.usage_ra_tls, + usage_server_auth: request.usage_server_auth, + usage_client_auth: request.usage_client_auth, + with_app_info: request.with_app_info, + not_before: request.not_before, + not_after: request.not_after, + }, + AttestationWire::MsgpackV1, + ) + .await?; + Ok(IssueCertResponse { + key: issued.key, + certificate_chain: issued.certificate_chain, + }) + } + + async fn get_key(self, request: GetKeyRequest) -> Result<GetKeyResponse> { + let (key, signature_chain) = self.derive(&request.domain, &request.algorithm)?; + Ok(GetKeyResponse { + key: key.secret(), + public_key: key.public_key(), + signature_chain, + }) + } + + async fn attest(self, request: AttestRequest) -> Result<AttestResponse> { + let report_data = pad64(&request.report_data).context("report data is too long")?; + // Generating a quote takes a global mutex and then blocks in an ioctl. + // On the async executor that parks a worker thread for the duration and + // stalls every other connection this agent is serving. + let state = self.state.clone(); + let attestation = tokio::task::spawn_blocking(move || { + state.attest_cvm(report_data, AttestationWire::MsgpackV1) + }) + .await + .context("the attestation task panicked")??; + Ok(AttestResponse { + attestation, + boottime_gpu_evidence: boottime_gpu_evidence( + request.include_boottime_gpu_evidence, + Path::new(GPU_ATTESTATION_OUTPUT), + ), + }) + } + + async fn attest_gpu(self, request: AttestGpuRequest) -> Result<AttestGpuResponse> { + let evidence = self + .state + .gpu_attestor() + .attest(&request.nonce) + .await + .context("GPU attestation failed")?; + Ok(AttestGpuResponse { + bundles: vec![GpuEvidenceBundle { + vendor: GPU_VENDOR.to_string(), + format: GPU_FORMAT_ON_DEMAND.to_string(), + evidence, + }], + }) + } + + /// Identity and configuration, never attestation. + /// + /// Ungated: the internal socket is reachable only by the application + /// itself, so there is nobody to hide from. The external surface applies + /// `public_tcbinfo`; see [`ExternalV1RpcHandler::info`]. + async fn info(self, _request: InfoRequest) -> Result<InfoResponse> { + info_response(&self.state, false) + } + + async fn version(self, _request: VersionRequest) -> Result<VersionResponse> { + Ok(version_response()) + } +} + +impl RpcCall<AppState> for V1RpcHandler { + type PrpcService = DstackGuestServer<Self>; + + fn construct(context: CallContext<'_, AppState>) -> Result<Self> { + Ok(V1RpcHandler { + state: context.state.clone(), + }) + } +} + +/// The v1 handler on the external listener. +/// +/// Reachable by anyone who can route to the CVM, so it serves no key material +/// and lets no caller choose what gets signed or attested. +pub struct ExternalV1RpcHandler { + state: AppState, +} + +impl ExternalV1RpcHandler { + #[cfg(test)] + pub(crate) fn new(state: AppState) -> Self { + Self { state } + } +} + +impl WorkerRpc for ExternalV1RpcHandler { + async fn info(self, _request: InfoRequest) -> Result<InfoResponse> { + let hide = !self.state.config().app_compose.public_tcbinfo; + info_response(&self.state, hide) + } + + async fn version(self, _request: VersionRequest) -> Result<VersionResponse> { + Ok(version_response()) + } + + async fn health(self, _request: HealthRequest) -> Result<HealthResponse> { + // One lock and one clone. Everything that costs anything happens on the + // agent's own timer in `health`, because this method is served on the + // publicly reachable listener and is polled by every gateway node in + // the cluster: any work done here is work an anonymous caller can ask + // for at an arbitrary rate, multiplied by the operator's fleet size. + let Some(monitor) = self.state.health() else { + // The app did not opt in, so it registered as "do not poll me" and + // no gateway asks. Anything that does ask gets the same answer the + // gateway would have assumed. + return Ok(HealthResponse { + healthy: true, + unhealthy: vec![], + error: String::new(), + }); + }; + // Deliberately infallible: see `HealthResponse.error`. The gateway + // counts every failed poll alike, so a failure to see the app has to + // come back as a verdict -- raised as an RPC error it would be lumped + // in with an unreachable agent and the reason would never reach the + // operator. + let verdict = monitor.report(); + Ok(HealthResponse { + healthy: verdict.healthy, + unhealthy: verdict.unhealthy, + error: verdict.error, + }) + } +} + +impl RpcCall<AppState> for ExternalV1RpcHandler { + type PrpcService = WorkerServer<Self>; + + fn construct(context: CallContext<'_, AppState>) -> Result<Self> { + Ok(ExternalV1RpcHandler { + state: context.state.clone(), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rpc_service::get_info; + use crate::rpc_service::tests::setup_test_state; + use dstack_guest_agent_rpc::v0::dstack_guest_server::DstackGuestRpc as DstackGuestV0Rpc; + use dstack_guest_agent_rpc::v0::GetKeyArgs; + use k256::ecdsa::Signature as K256Signature; + use std::io::Write as _; + + async fn state() -> (AppState, tempfile::NamedTempFile) { + setup_test_state().await + } + + fn get_key_request(domain: &str, algorithm: &str) -> GetKeyRequest { + GetKeyRequest { + domain: domain.to_string(), + algorithm: algorithm.to_string(), + } + } + + /// The fixture's app root key is what `keys::tests` committed its vectors + /// against. If the two drift, the vectors stop describing what this + /// handler serves and every assertion built on them goes quiet. + #[tokio::test] + async fn the_fixture_root_key_matches_the_committed_vectors() { + let (state, _guard) = state().await; + assert_eq!( + hex::encode(state.app_root_k256_key()), + "1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b" + ); + } + + #[tokio::test] + async fn get_key_returns_a_two_link_chain_for_both_algorithms() { + for algorithm in ["secp256k1", "ed25519"] { + let (state, _guard) = state().await; + let key = V1RpcHandler::new(state) + .get_key(get_key_request("storage-encryption", algorithm)) + .await + .unwrap(); + + assert_eq!(key.key.len(), 32); + assert_eq!(key.signature_chain.len(), 2); + // r || s || recovery id. + assert_eq!(key.signature_chain[0].len(), 65); + } + } + + /// The chain's first link verifies under the app root key, over the claim + /// a relying party rebuilds from `(domain, algorithm, public_key)`. + #[tokio::test] + async fn the_chain_claim_verifies_under_the_app_root_key() { + use k256::ecdsa::{RecoveryId, SigningKey, VerifyingKey}; + use sha3::{Digest as _, Keccak256}; + + let (state, _guard) = state().await; + let key = V1RpcHandler::new(state.clone()) + .get_key(get_key_request("storage-encryption", "secp256k1")) + .await + .unwrap(); + + let claim = ra_tls::api_v1::key_claim( + keys::Algorithm::Secp256k1, + "storage-encryption", + &key.public_key, + ) + .unwrap(); + let link = &key.signature_chain[0]; + let recovered = VerifyingKey::recover_from_digest( + Keccak256::new_with_prefix(&claim), + &K256Signature::from_slice(&link[..64]).unwrap(), + RecoveryId::from_byte(link[64]).unwrap(), + ) + .unwrap(); + + let app_root = *SigningKey::from_slice(state.app_root_k256_key()) + .unwrap() + .verifying_key(); + assert_eq!(recovered, app_root); + } + + /// The public key in the response is the one the returned private key + /// actually has. An app that signs locally with `key` must land on + /// `public_key`, which is what the chain vouches for. + #[tokio::test] + async fn the_public_key_belongs_to_the_returned_private_key() { + let (state, _guard) = state().await; + + let secp = V1RpcHandler::new(state.clone()) + .get_key(get_key_request("storage-encryption", "secp256k1")) + .await + .unwrap(); + let derived = k256::ecdsa::SigningKey::from_slice(&secp.key).unwrap(); + assert_eq!( + derived.verifying_key().to_sec1_bytes().to_vec(), + secp.public_key + ); + + let ed = V1RpcHandler::new(state) + .get_key(get_key_request("storage-encryption", "ed25519")) + .await + .unwrap(); + let seed: [u8; 32] = ed.key.as_slice().try_into().unwrap(); + let derived = ed25519_dalek::SigningKey::from_bytes(&seed); + assert_eq!(derived.verifying_key().to_bytes().to_vec(), ed.public_key); + } + + #[tokio::test] + async fn rejects_an_empty_or_unknown_algorithm() { + let (state, _guard) = state().await; + for algorithm in ["", "k256", "rsa", "secp256k1_prehashed"] { + let result = V1RpcHandler::new(state.clone()) + .get_key(get_key_request("storage-encryption", algorithm)) + .await; + assert!(result.is_err(), "v1 accepted algorithm {algorithm:?}"); + } + } + + /// v1 keys are new key material. An app that reuses its v0 path as a v1 + /// domain gets a different key, and that is the migration contract, not a + /// bug to paper over. + #[tokio::test] + async fn v1_keys_differ_from_the_unversioned_keys_for_the_same_name() { + let (state, _guard) = state().await; + let v1 = V1RpcHandler::new(state.clone()) + .get_key(get_key_request("storage-encryption", "secp256k1")) + .await + .unwrap(); + let v0 = crate::rpc_service::InternalRpcHandler::new(state) + .get_key(GetKeyArgs { + path: "storage-encryption".to_string(), + purpose: "signing".to_string(), + algorithm: "secp256k1".to_string(), + }) + .await + .unwrap(); + + assert_ne!(v1.key, v0.key); + // The second link is the KMS's, produced outside the agent, so both + // surfaces pass through the same bytes. + assert_eq!(v1.signature_chain[1], v0.signature_chain[1]); + assert_ne!(v1.signature_chain[0], v0.signature_chain[0]); + } + + /// The unversioned handler is untouched by any of this: same fixture, same + /// path, the v0.5.x answer. + #[tokio::test] + async fn the_unversioned_surface_still_serves_its_own_key() { + let (state, _guard) = state().await; + // v0 defaults an empty algorithm to secp256k1 and derives from the + // path alone; v1 rejects the same request outright. + let expected = ra_tls::kdf::derive_key(state.app_root_k256_key(), &[b"test"], 32).unwrap(); + let v0 = crate::rpc_service::InternalRpcHandler::new(state) + .get_key(GetKeyArgs { + path: "test".to_string(), + purpose: "signing".to_string(), + algorithm: String::new(), + }) + .await + .unwrap(); + assert_eq!(v0.key, expected); + } + + /// `Info` must not attest. Decoding identity costs a hardware quote and an + /// RTMR replay under a global lock, and the v1 `Worker.Info` is served on + /// the public listener -- so doing it per call hands any caller that can + /// route to the CVM a way to monopolise the attestation path. + /// + /// One decoded `AppIdentity`, shared by pointer, is what says it is cached. + #[tokio::test] + async fn info_decodes_identity_at_most_once() { + let (state, _guard) = state().await; + + let first = state.identity().unwrap(); + for _ in 0..8 { + V1RpcHandler::new(state.clone()) + .info(InfoRequest {}) + .await + .unwrap(); + ExternalV1RpcHandler::new(state.clone()) + .info(InfoRequest {}) + .await + .unwrap(); + } + let last = state.identity().unwrap(); + + assert!( + std::sync::Arc::ptr_eq(&first, &last), + "identity was decoded again; every Info call is generating a quote" + ); + } + + /// v1 `Info` reports identity and configuration, and nothing that only an + /// attestation should be trusted for. + #[tokio::test] + async fn info_reports_identity_and_configuration() { + let (state, _guard) = state().await; + let v1 = V1RpcHandler::new(state.clone()) + .info(InfoRequest {}) + .await + .unwrap(); + let v0 = get_info(&state, false).await.unwrap(); + + // The identity fields carry the same values the unversioned surface + // reports; only the shape changed. + assert_eq!(v1.app_id, v0.app_id); + assert_eq!(v1.instance_id, v0.instance_id); + assert_eq!(v1.device_id, v0.device_id); + assert_eq!(v1.mr_aggregated, v0.mr_aggregated); + assert_eq!(v1.os_image_hash, v0.os_image_hash); + assert_eq!(v1.compose_hash, v0.compose_hash); + assert_eq!(v1.app_name, v0.app_name); + assert_eq!(v1.vm_config, v0.vm_config); + assert_eq!(v1.key_provider_info, v0.key_provider_info); + assert_eq!(v1.cloud_vendor, v0.cloud_vendor); + assert_eq!(v1.cloud_product, v0.cloud_product); + + // The app-compose document is served directly rather than nested in a + // JSON string, which is the one thing v0 could not do. + assert_eq!(v1.app_compose, state.config().app_compose.raw); + } + + /// GPU evidence rides on `Attest`, which is the only way to get it now, + /// and it comes back in the same bundle shape `AttestGpu` uses. + #[tokio::test] + async fn attest_returns_boot_time_gpu_evidence_only_when_asked() { + let mut output = tempfile::NamedTempFile::new().unwrap(); + let evidence = r#"{"result_code":0,"claims":[]}"#; + output.write_all(evidence.as_bytes()).unwrap(); + output.flush().unwrap(); + + let bundles = boottime_gpu_evidence(true, output.path()); + assert_eq!(bundles.len(), 1); + assert_eq!(bundles[0].vendor, "nvidia"); + // The tag that separates the boot record from on-demand collection. + assert_eq!(bundles[0].format, "nvidia-nvattest-boottime-json-v1"); + assert_ne!(bundles[0].format, GPU_FORMAT_ON_DEMAND); + // Byte-exact: the event-log binding is sha256 over precisely these + // bytes, so anything that reformats the JSON breaks verification. + assert_eq!(bundles[0].evidence, evidence.as_bytes()); + + assert!(boottime_gpu_evidence(false, output.path()).is_empty()); + } + + /// No GPU output is the empty list, not a bundle carrying nothing. + #[tokio::test] + async fn attest_returns_no_bundle_when_there_is_no_gpu_output() { + let dir = tempfile::tempdir().unwrap(); + assert!(boottime_gpu_evidence(true, &dir.path().join("missing")).is_empty()); + } + + #[test] + fn reads_gpu_attestation_output_verbatim() { + let mut output = tempfile::NamedTempFile::new().unwrap(); + // Trailing newline and internal spacing included on purpose: this is + // read byte for byte, never normalised. + let attestation = "{\"result_code\": 0, \"claims\": []}\n"; + output.write_all(attestation.as_bytes()).unwrap(); + output.flush().unwrap(); + + assert_eq!( + read_gpu_attestation(output.path()).unwrap(), + attestation.as_bytes() + ); + } + + #[test] + fn missing_gpu_attestation_output_reads_as_none() { + let dir = tempfile::tempdir().unwrap(); + assert!(read_gpu_attestation(&dir.path().join("missing")).is_none()); + } + + /// `Attest` is v1's only CVM attestation entry point, and the versioned + /// attestation it returns carries the report data the caller asked for. + #[tokio::test] + async fn attest_reports_the_padded_report_data() { + use ra_tls::attestation::VersionedAttestation; + + let (state, _guard) = state().await; + let response = V1RpcHandler::new(state) + .attest(AttestRequest { + report_data: b"hello".to_vec(), + include_boottime_gpu_evidence: false, + }) + .await + .unwrap(); + + let report_data = VersionedAttestation::from_bytes(&response.attestation) + .unwrap() + .into_v1() + .report_data() + .unwrap(); + assert_eq!(&report_data[..5], b"hello"); + assert!(report_data[5..].iter().all(|b| *b == 0)); + } + + /// v1 `Attest` commits to the MessagePack V1 schema. The fixture platform + /// produces the legacy SCALE form (as `RealPlatform` does for a TDX CVM with + /// V1 runtime events), so this pins the re-encoding rather than a platform + /// that already happened to emit V1. + #[tokio::test] + async fn attest_always_returns_msgpack() { + use ra_tls::attestation::VersionedAttestation; + + let (state, _guard) = state().await; + let legacy = state + .attest_cvm([0u8; 64], AttestationWire::Legacy) + .unwrap(); + assert_eq!( + legacy.first(), + Some(&0x00), + "the fixture must exercise the legacy-to-V1 re-encoding" + ); + + let response = V1RpcHandler::new(state) + .attest(AttestRequest { + report_data: b"hello".to_vec(), + include_boottime_gpu_evidence: false, + }) + .await + .unwrap(); + + let first = *response.attestation.first().unwrap(); + assert!( + matches!(first, 0x80..=0x8f | 0xde | 0xdf), + "expected a MessagePack map, got first byte {first:#04x}" + ); + let attestation = VersionedAttestation::from_bytes(&response.attestation).unwrap(); + let VersionedAttestation::V1 { attestation } = attestation else { + panic!("v1 Attest must return the V1 schema"); + }; + assert_eq!(&attestation.report_data().unwrap()[..5], b"hello"); + assert!(attestation.platform.tdx_quote().is_some()); + } + + /// End to end: the exact bytes v1 `Attest` hands out, encoded through + /// `AttestationWire::MsgpackV1`, pass full attestation verification -- the + /// DCAP quote, the RTMR3 replay and the report data -- and verify to the + /// same identity as the frozen v0 `Attest` output in the legacy form. + /// + /// The test platform patches the requested report data into the fixture + /// quote, which breaks its signature for any other value. Asking for the + /// fixture's own report data leaves the captured quote intact. + /// + /// Like the `dstack-verifier` fixture tests, this fetches DCAP collateral + /// from the default PCCS, so it needs network access. + #[tokio::test] + async fn attest_output_passes_attestation_verification() { + use crate::rpc_service::InternalRpcHandler; + use dstack_guest_agent_rpc::v0::{dstack_guest_server::DstackGuestRpc as _, RawQuoteArgs}; + use ra_tls::attestation::{AttestationVerifier, VersionedAttestation}; + + let report_data = + VersionedAttestation::from_bytes(include_bytes!("../fixtures/attestation.bin")) + .unwrap() + .into_v1() + .report_data() + .unwrap(); + let verifier = AttestationVerifier::new_prod(None).unwrap(); + + let (state, _guard) = state().await; + let v1 = V1RpcHandler::new(state.clone()) + .attest(AttestRequest { + report_data: report_data.to_vec(), + include_boottime_gpu_evidence: false, + }) + .await + .unwrap() + .attestation; + let v0 = InternalRpcHandler::new(state) + .attest(RawQuoteArgs { + report_data: report_data.to_vec(), + }) + .await + .unwrap() + .attestation; + + let mut identities = Vec::new(); + for (surface, bytes, expect_v1) in [("v1", &v1, true), ("v0", &v0, false)] { + let attestation = VersionedAttestation::from_bytes(bytes).unwrap(); + assert_eq!( + matches!(attestation, VersionedAttestation::V1 { .. }), + expect_v1, + "{surface} Attest returned the wrong wire form" + ); + let verified = attestation + .into_v1() + .verify(&verifier) + .await + .unwrap_or_else(|err| panic!("{surface} Attest output must verify: {err:#}")); + assert_eq!(verified.report_data, report_data, "{surface}"); + let app = verified.decode_app_info(false).unwrap(); + identities.push(( + app.app_id, + app.compose_hash, + app.instance_id, + app.mr_aggregated, + app.os_image_hash, + )); + } + assert_eq!( + identities[0], identities[1], + "v1 and v0 Attest verified to different identities" + ); + + // Control: any other report data is patched into the quote body the + // signature covers, so the same path must now reject it. Without this + // the test cannot tell a verification that ran from one that did not. + let mut other = report_data; + other[0] ^= 1; + let (forge_state, _forge_guard) = setup_test_state().await; + let forged = V1RpcHandler::new(forge_state) + .attest(AttestRequest { + report_data: other.to_vec(), + include_boottime_gpu_evidence: false, + }) + .await + .unwrap() + .attestation; + assert!( + VersionedAttestation::from_bytes(&forged) + .unwrap() + .into_v1() + .verify(&verifier) + .await + .is_err(), + "a quote whose report data no longer matches its signature must not verify" + ); + } + + #[tokio::test] + async fn rejects_report_data_longer_than_64_bytes() { + let (state, _guard) = state().await; + let err = V1RpcHandler::new(state) + .attest(AttestRequest { + report_data: vec![0; 65], + include_boottime_gpu_evidence: false, + }) + .await + .unwrap_err() + .to_string(); + assert!(err.contains("too long"), "{err}"); + } + + #[tokio::test] + async fn version_answers() { + let (state, _guard) = state().await; + let response = V1RpcHandler::new(state) + .version(VersionRequest {}) + .await + .unwrap(); + assert!(!response.version.is_empty()); + } + + /// The external surface honours the app's `public_tcbinfo` choice; the + /// internal one has nobody to hide from. + #[tokio::test] + async fn the_external_surface_hides_documents_unless_the_app_opted_in() { + let (state, _guard) = state().await; + assert!( + !state.config().app_compose.public_tcbinfo, + "the fixture must default to private for this test to mean anything" + ); + + let external = ExternalV1RpcHandler::new(state.clone()) + .info(InfoRequest {}) + .await + .unwrap(); + assert_eq!(external.app_compose, ""); + assert_eq!(external.vm_config, ""); + assert_eq!(external.key_provider_info, ""); + + // Identity and the measurement hashes stay visible, which is the line + // the unversioned `Worker.Info` drew too. + let internal = V1RpcHandler::new(state).info(InfoRequest {}).await.unwrap(); + assert_eq!(external.app_id, internal.app_id); + assert_eq!(external.instance_id, internal.instance_id); + assert_eq!(external.compose_hash, internal.compose_hash); + assert_eq!(external.os_image_hash, internal.os_image_hash); + assert_eq!(external.mr_aggregated, internal.mr_aggregated); + } + + /// An app that opted in gets the full response on the external surface. + #[tokio::test] + async fn the_external_surface_serves_documents_when_the_app_opted_in() { + let (state, _guard) = + crate::rpc_service::tests::setup_test_state_with_public_tcbinfo().await; + let external = ExternalV1RpcHandler::new(state.clone()) + .info(InfoRequest {}) + .await + .unwrap(); + assert_eq!(external.app_compose, state.config().app_compose.raw); + } + + /// An app that never opted into health gating is never polled, and anything + /// that asks anyway gets the answer the gateway would have assumed. + #[tokio::test] + async fn health_fails_open_for_an_app_that_did_not_opt_in() { + let (state, _guard) = state().await; + let response = ExternalV1RpcHandler::new(state) + .health(HealthRequest {}) + .await + .unwrap(); + assert!(response.healthy); + assert!(response.unhealthy.is_empty()); + assert!(response.error.is_empty()); + } + + /// Route-level check without a live socket: the two generated dispatchers + /// own disjoint method tables, which is what makes mounting one at `/` and + /// the other at `/v1` a version selector rather than a name collision. + #[test] + fn the_two_surfaces_expose_different_method_sets() { + // Both packages call the service `DstackGuest` now, so the module path + // is what disambiguates them -- which is the point of the v0/v1 split. + use dstack_guest_agent_rpc::v0::dstack_guest_server::DstackGuestServer as V0Server; + use dstack_guest_agent_rpc::v1::dstack_guest_server::DstackGuestServer as V1Server; + + let v0 = V0Server::<crate::rpc_service::InternalRpcHandler>::supported_methods(); + let v1 = V1Server::<V1RpcHandler>::supported_methods(); + + assert_eq!( + v1, + &[ + "IssueCert", + "GetKey", + "Attest", + "AttestGpu", + "Info", + "Version" + ], + "the v1 surface changed" + ); + + // The unversioned surface is closed at exactly v0.5.11. + assert_eq!( + v0, + &[ + "GetTlsKey", + "GetKey", + "GetQuote", + "Attest", + "EmitEvent", + "Info", + "Sign", + "Verify", + "Version" + ], + "the unversioned surface is frozen at the v0.5.11 method set" + ); + + // Everything v1 deliberately does not serve. `Sign` and `Verify` are + // pure computation over a key the caller can already fetch, `GetQuote` + // is the TDX-only channel `Attest` subsumes, and RTMR3 is + // system-owned. All four stay on the frozen surface. + for dropped in ["Sign", "Verify", "GetQuote", "EmitEvent"] { + assert!(!v1.contains(&dropped), "v1 must not serve {dropped}"); + } + + // v0 keeps the old name for what v1 calls `IssueCert`. + assert!(!v1.contains(&"GetTlsKey")); + + // Never-released `next` additions that now live only in v1, or nowhere. + assert!(!v0.contains(&"AttestGpu")); + assert!(v1.contains(&"AttestGpu")); + assert!(!v0.contains(&"GpuInfo") && !v1.contains(&"GpuInfo")); + } + + /// The external pair, checked the same way. + #[test] + fn the_two_external_surfaces_expose_different_method_sets() { + use dstack_guest_agent_rpc::v0::worker_server::WorkerServer as WorkerV0Server; + use dstack_guest_agent_rpc::v1::worker_server::WorkerServer as WorkerV1Server; + + let v0 = WorkerV0Server::<crate::rpc_service::ExternalRpcHandler>::supported_methods(); + let v1 = WorkerV1Server::<ExternalV1RpcHandler>::supported_methods(); + + // Closed at v0.5.11. `Health` is post-0.5.11 and never released, so it + // lives only on v1. There is no v1 `AttestAppKey`: it would attest the + // v0-KDF `vms` key, which no v1 `GetKey(domain, algorithm)` can return, + // so a pure-v1 app could never hold the attested private key. A v1 app + // attests its own key through `/v1/Attest`. + assert_eq!( + v0, + &["Info", "Version", "GetAttestationForAppKey"], + "the unversioned external surface is frozen at the v0.5.11 method set" + ); + assert_eq!(v1, &["Info", "Version", "Health"]); + } + + /// `IssueCert` must reject an inverted validity window, and must do it + /// before generating a key or calling the KMS. + /// + /// The check lives in the shared `issue_cert_for_request`, which the frozen + /// `GetTlsKey` also calls, so nothing here would fail if the v1 handler + /// stopped routing through it -- which is exactly why the assertion is made + /// against the v1 handler rather than against the validator alone. + #[tokio::test] + async fn issue_cert_rejects_an_inverted_validity_window() { + let (state, _guard) = state().await; + let err = V1RpcHandler::new(state) + .issue_cert(IssueCertRequest { + subject: "example".to_string(), + not_before: Some(2_000), + not_after: Some(1_000), + ..Default::default() + }) + .await + .unwrap_err() + .to_string(); + assert!( + err.contains("not_before must be earlier than not_after"), + "{err}" + ); + } + + /// `IssueCert` commits to MessagePack as `Attest` does: the attestation + /// goes into the CSR as V1, and the signer embeds it in the certificate in + /// the form it received. The fixture platform produces the legacy form, so + /// this pins the re-encoding (see + /// `rpc_service::tests::v0_get_tls_key_keeps_the_legacy_wire_format`). + #[tokio::test] + async fn issue_cert_embeds_a_msgpack_attestation() { + use crate::rpc_service::tests::embedded_attestation; + use ra_tls::attestation::VersionedAttestation; + + let (state, _guard) = state().await; + let issued = V1RpcHandler::new(state) + .issue_cert(IssueCertRequest { + subject: "example".to_string(), + usage_ra_tls: true, + ..Default::default() + }) + .await + .unwrap(); + let attestation = embedded_attestation(&issued.certificate_chain[0]); + assert!( + matches!(attestation, VersionedAttestation::V1 { .. }), + "v1 IssueCert must embed the V1 schema" + ); + } + + /// `AttestGpu` routes through the attestor, which validates the nonce + /// length before it touches a device. Pinned through the handler so a + /// future refactor cannot answer without asking the attestor at all. + /// + /// Formatted with `{:#}`, because the handler adds a context line and the + /// reason sits under it. That is also how the caller reads it: `ra_rpc` + /// encodes an error as `format!("{error:#}")`, so asserting on the flat + /// `to_string()` would test something no client ever sees. + #[tokio::test] + async fn attest_gpu_rejects_a_wrong_length_nonce() { + let (state, _guard) = state().await; + let err = V1RpcHandler::new(state) + .attest_gpu(AttestGpuRequest { + nonce: vec![0u8; 16], + }) + .await + .unwrap_err(); + assert_eq!( + ra_rpc::code_of(&err), + None, + "a malformed nonce is the caller's fault, so it keeps the default 400" + ); + let err = format!("{err:#}"); + assert!(err.contains("exactly 32 bytes"), "{err}"); + } + + /// An image that ships no nvattest must not answer like a malformed + /// request: the call is well-formed and no other call would succeed, so + /// the caller has to be able to tell "stop" from "try something else". + /// + /// Asserted through the handler because that is where `.context(..)` wraps + /// the attestor's error. `code_of` walks the whole chain for exactly this + /// reason -- a context layer that buried the code would silently put the + /// answer back to 400, and the message would still read correctly. + #[tokio::test] + async fn attest_gpu_reports_an_image_without_gpu_support_as_not_implemented() { + let (state, _guard) = state().await; + let err = V1RpcHandler::new(state) + .attest_gpu(AttestGpuRequest { + nonce: vec![0u8; 32], + }) + .await + .unwrap_err(); + assert_eq!(ra_rpc::code_of(&err), Some(501), "{err:#}"); + } + + /// The on-demand format tag, pinned like its boot-time counterpart. + /// + /// A consumer selects its verifier on `(vendor, format)`, so these two + /// strings are wire contract: the boot-time tag is already pinned by + /// `attest_returns_boot_time_gpu_evidence_only_when_asked`, and this is the + /// other half. Collecting real evidence needs a GPU, so the tags -- not a + /// live bundle -- are what a test can hold still. + #[test] + fn the_on_demand_gpu_format_tag_is_the_specified_one() { + assert_eq!(GPU_VENDOR, "nvidia"); + assert_eq!( + GPU_FORMAT_ON_DEMAND, + "nvidia-nvattest-collect-evidence-json-v1" + ); + assert_ne!(GPU_FORMAT_ON_DEMAND, GPU_FORMAT_BOOTTIME); + } +} diff --git a/dstack/guest-agent/src/rpc_service_v1/keys.rs b/dstack/guest-agent/src/rpc_service_v1/keys.rs new file mode 100644 index 000000000..7163b674e --- /dev/null +++ b/dstack/guest-agent/src/rpc_service_v1/keys.rs @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! The v1 application key, assembled from the shared normative primitives. +//! +//! The encodings themselves -- the salt, the context tags, the length-prefixed +//! `info` and claim, and the chain-link envelope -- live in +//! [`ra_tls::api_v1`], so the agent, the verifier and the SDK share one +//! definition instead of three transcriptions of the specification prose. What +//! is left here is turning derived bytes into a usable key pair. + +use anyhow::{anyhow, Context, Result}; +use ed25519_dalek::SigningKey as Ed25519SigningKey; +use k256::ecdsa::SigningKey; +use ra_tls::api_v1::{derive_app_key, key_claim, sign_recoverable_keccak256}; + +pub(crate) use ra_tls::api_v1::KeyAlgorithm as Algorithm; + +/// An application key derived for one `(domain, algorithm)` pair. +pub(crate) struct AppKey { + algorithm: Algorithm, + domain: String, + secret: [u8; 32], + public_key: Vec<u8>, +} + +impl AppKey { + /// Derive the key for `(domain, algorithm)` from the app root key. + /// + /// Flat, not hierarchical: `a/b` is an opaque domain string like any other, + /// unrelated to `a`, and no key here derives another. + pub(crate) fn derive(app_root_key: &[u8], domain: &str, algorithm: Algorithm) -> Result<Self> { + let secret = derive_app_key(app_root_key, domain, algorithm)?; + let public_key = match algorithm { + // Rejects a scalar that is zero or at least the group order. That + // is a ~2^-128 event for one domain and the caller can just pick + // another one, so failing is better than folding the scalar into + // range and quietly landing two domains on one key. + Algorithm::Secp256k1 => SigningKey::from_slice(&secret) + .context("derived secp256k1 key is not a valid scalar")? + .verifying_key() + .to_sec1_bytes() + .to_vec(), + Algorithm::Ed25519 => Ed25519SigningKey::from_bytes(&secret) + .verifying_key() + .to_bytes() + .to_vec(), + }; + Ok(Self { + algorithm, + domain: domain.to_string(), + secret, + public_key, + }) + } + + /// The raw 32-byte private key. + pub(crate) fn secret(&self) -> Vec<u8> { + self.secret.to_vec() + } + + /// SEC1 compressed for secp256k1, 32 raw bytes for ed25519. + pub(crate) fn public_key(&self) -> Vec<u8> { + self.public_key.clone() + } + + /// The app root key's signature over this key's claim: the first link of + /// the signature chain. + /// + /// Takes the parsed key rather than its bytes, so the caller can share one + /// across requests instead of re-parsing a scalar per `GetKey`. + pub(crate) fn claim_signature(&self, app_root_key: &SigningKey) -> Result<Vec<u8>> { + let claim = key_claim(self.algorithm, &self.domain, &self.public_key)?; + sign_recoverable_keccak256(app_root_key, &claim) + .map_err(|err| anyhow!("failed to sign the key claim: {err:#}")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The app root key `ra_tls::api_v1` committed its vectors against. + const TEST_APP_ROOT_KEY: [u8; 32] = [ + 0x1A, 0x2B, 0x3C, 0x4D, 0x5E, 0x6F, 0x7A, 0x8B, 0x9C, 0x0D, 0x1E, 0x2F, 0x3A, 0x4B, 0x5C, + 0x6D, 0x7E, 0x8F, 0x9A, 0x0B, 0x1C, 0x2D, 0x3E, 0x4F, 0x5A, 0x6B, 0x7C, 0x8D, 0x9E, 0x0F, + 0x1A, 0x2B, + ]; + + fn derive(domain: &str, algorithm: Algorithm) -> AppKey { + AppKey::derive(&TEST_APP_ROOT_KEY, domain, algorithm).unwrap() + } + + /// The public key vectors, checked through the type the handler actually + /// uses. The private-key and encoding vectors are pinned next to the + /// primitives in `ra_tls::api_v1`. + #[test] + fn derives_the_committed_public_key_vectors() { + let vectors = [ + ( + "", + Algorithm::Secp256k1, + "0377c7fb050db181d392266a3cee9adb2901c6d665f11bac68be5457f577ba4908", + ), + ( + "", + Algorithm::Ed25519, + "a3dc149fd5b765eab2eb7d3174fa939e39386898f10b15b7b146f6f1358ecf2a", + ), + ( + "storage-encryption", + Algorithm::Secp256k1, + "03d962450a41748021c8b02787ac36ce642ff0ae25f4c55019eb527e1112cfd764", + ), + ( + "storage-encryption", + Algorithm::Ed25519, + "2380c4a33a60b60613fa43866421e5b96eb8dcde211317200fd0d41e7e491288", + ), + ( + "a/b/c", + Algorithm::Secp256k1, + "02e9b1a61b6d70aa9b241753828c316bf90e33e77b2e113f9ba75a8b6dc3cde5c1", + ), + ( + "k\u{0}:ey", + Algorithm::Ed25519, + "c833107822b003ff5675b33b90b151d4315c3ab9162b17d876e8dffde41abf9b", + ), + ]; + for (domain, algorithm, expected) in vectors { + assert_eq!( + hex::encode(derive(domain, algorithm).public_key()), + expected, + "v1 public key vector changed for ({domain:?}, {})", + algorithm.name() + ); + } + } + + #[test] + fn the_public_key_lengths_are_the_specified_ones() { + assert_eq!( + derive("storage-encryption", Algorithm::Secp256k1) + .public_key() + .len(), + 33 + ); + assert_eq!( + derive("storage-encryption", Algorithm::Ed25519) + .public_key() + .len(), + 32 + ); + } + + #[test] + fn the_two_algorithms_never_share_key_material() { + for domain in ["", "storage-encryption", "a/b/c"] { + assert_ne!( + derive(domain, Algorithm::Secp256k1).secret(), + derive(domain, Algorithm::Ed25519).secret(), + "cross-algorithm key reuse at {domain:?}" + ); + } + } + + #[test] + fn the_claim_signature_verifies_under_the_app_root_key() { + use k256::ecdsa::{RecoveryId, Signature, VerifyingKey}; + use sha3::{Digest as _, Keccak256}; + + let key = derive("storage-encryption", Algorithm::Secp256k1); + let app_root = SigningKey::from_slice(&TEST_APP_ROOT_KEY).unwrap(); + let link = key.claim_signature(&app_root).unwrap(); + assert_eq!(link.len(), 65); + + let claim = key_claim( + Algorithm::Secp256k1, + "storage-encryption", + &key.public_key(), + ) + .unwrap(); + let recovered = VerifyingKey::recover_from_digest( + Keccak256::new_with_prefix(&claim), + &Signature::from_slice(&link[..64]).unwrap(), + RecoveryId::from_byte(link[64]).unwrap(), + ) + .unwrap(); + assert_eq!(&recovered, app_root.verifying_key()); + } +} diff --git a/dstack/guest-agent/src/server.rs b/dstack/guest-agent/src/server.rs index e6cc8bd79..0021f4f9b 100644 --- a/dstack/guest-agent/src/server.rs +++ b/dstack/guest-agent/src/server.rs @@ -8,6 +8,7 @@ use crate::config::BindAddr; use crate::guest_api_service::GuestApiHandler; use crate::http_routes; use crate::rpc_service::{AppState, ExternalRpcHandler, InternalRpcHandler, InternalRpcHandlerV0}; +use crate::rpc_service_v1::{ExternalV1RpcHandler, V1RpcHandler}; use crate::socket_activation::{ActivatedSockets, ActivatedUnixListener}; use anyhow::{anyhow, Context, Result}; use ra_rpc::rocket_helper::UnixPeerCredListener; @@ -15,6 +16,7 @@ use rocket::{ fairing::AdHoc, figment::Figment, listener::{unix::UnixListener, Bind, DefaultListener, Endpoint}, + Build, Rocket, }; use rocket_vsock_listener::VsockListener; use sd_notify::{notify as sd_notify, NotifyState}; @@ -22,7 +24,7 @@ use tokio::sync::oneshot; use tracing::{error, info}; pub fn app_version() -> String { - format!("v{} ({})", crate::CARGO_PKG_VERSION, crate::GIT_REV) + dstack_build_info::app_version!() } async fn run_internal_v0( @@ -80,15 +82,57 @@ async fn run_internal_v0( Ok(()) } +/// Mount everything the internal socket serves. +/// +/// `/v0` is the name of the frozen v0.5.11 surface and `/v1` is the current +/// one. `/` is the same frozen surface under its historical path, kept as a +/// compatibility alias so a pre-0.6 client keeps working unchanged -- it is the +/// identical handler, not a copy, so the two paths cannot drift. +/// +/// Selection is by URL path alone: no header negotiation, no default-version +/// redirect, so a caller's URL is the whole record of which contract it asked +/// for. +/// +/// Factored out of `run_internal` so a test can exercise the real mount table +/// rather than a restatement of it. +fn mount_internal(rocket: Rocket<Build>) -> Rocket<Build> { + rocket + .mount("/", ra_rpc::prpc_routes!(AppState, InternalRpcHandler)) + .mount("/v0", ra_rpc::prpc_routes!(AppState, InternalRpcHandler)) + .mount("/v1", ra_rpc::prpc_routes!(AppState, V1RpcHandler)) +} + +/// Mount the pRPC services the external listener serves. +/// +/// Same scheme as the internal socket, one level down: `/prpc/v0` is the frozen +/// v0.5.11 `Worker`, `/prpc/v1` is the v1 `Worker`, and `/prpc` is the frozen +/// surface under its historical path. +/// +/// The `trim` on the frozen mounts strips the service name a pre-0.6 client +/// prefixes, so `/prpc/Worker.Info` and `/prpc/Info` both land on `Info`. +fn mount_external(rocket: Rocket<Build>) -> Rocket<Build> { + rocket + .mount( + "/prpc", + ra_rpc::prpc_routes!(AppState, ExternalRpcHandler, trim: "Worker."), + ) + .mount( + "/prpc/v0", + ra_rpc::prpc_routes!(AppState, ExternalRpcHandler, trim: "Worker."), + ) + .mount( + "/prpc/v1", + ra_rpc::prpc_routes!(AppState, ExternalV1RpcHandler), + ) +} + async fn run_internal( state: AppState, figment: Figment, activated_socket: Option<StdUnixListener>, sock_ready_tx: oneshot::Sender<()>, ) -> Result<()> { - let rocket = rocket::custom(figment) - .mount("/", ra_rpc::prpc_routes!(AppState, InternalRpcHandler)) - .manage(state); + let rocket = mount_internal(rocket::custom(figment)).manage(state); let ignite = rocket .ignite() .await @@ -133,12 +177,8 @@ async fn run_internal( } async fn run_external(state: AppState, figment: Figment) -> Result<()> { - let rocket = rocket::custom(figment) + let rocket = mount_external(rocket::custom(figment)) .mount("/", http_routes::external_routes(state.config())) - .mount( - "/prpc", - ra_rpc::prpc_routes!(AppState, ExternalRpcHandler, trim: "Worker."), - ) .attach(AdHoc::on_response("Add app version header", |_req, res| { Box::pin(async move { res.set_raw_header("X-App-Version", app_version()); @@ -257,3 +297,132 @@ pub async fn run(state: AppState, figment: Figment, watchdog: bool) -> Result<() ); Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::rpc_service::tests::setup_test_state; + use rocket::local::asynchronous::Client; + + /// One agent, serving a whole mount table, for the life of a test. + /// + /// A `Client` per request would compare two different rocket instances and + /// two different `AppState`s; the alias property under test is that `/v0` + /// and `/` reach the *same* handler on the *same* instance. + async fn client( + mount: fn(Rocket<Build>) -> Rocket<Build>, + ) -> (Client, tempfile::NamedTempFile) { + let (state, guard) = setup_test_state().await; + let client = Client::tracked(mount(rocket::build()).manage(state)) + .await + .expect("rocket failed to ignite"); + (client, guard) + } + + async fn get(client: &Client, path: &str) -> (u16, String) { + let response = client.get(path).dispatch().await; + let status = response.status().code; + (status, response.into_string().await.unwrap_or_default()) + } + + /// The frozen surface answers identically on `/v0` and on the unversioned + /// path it has always had. + /// + /// The alias is what lets a pre-0.6 client keep working, so it has to stay + /// the same handler rather than a second one that happens to agree today. + #[tokio::test] + async fn the_internal_v0_mount_is_an_alias_for_the_unversioned_path() { + let (client, _guard) = client(mount_internal).await; + let (unversioned_status, unversioned) = get(&client, "/Version").await; + let (v0_status, v0) = get(&client, "/v0/Version").await; + + assert_eq!(unversioned_status, 200, "{unversioned}"); + assert_eq!(v0_status, 200, "{v0}"); + assert_eq!(unversioned, v0); + assert!(v0.contains("version"), "{v0}"); + } + + /// Same on the external listener, one level down. + #[tokio::test] + async fn the_external_v0_mount_is_an_alias_for_the_unversioned_path() { + let (client, _guard) = client(mount_external).await; + let (unversioned_status, unversioned) = get(&client, "/prpc/Version").await; + let (v0_status, v0) = get(&client, "/prpc/v0/Version").await; + + assert_eq!(unversioned_status, 200, "{unversioned}"); + assert_eq!(v0_status, 200, "{v0}"); + assert_eq!(unversioned, v0); + } + + /// A pre-0.6 client prefixes the service name. That has to keep working on + /// the alias and on `/v0`. + #[tokio::test] + async fn the_external_mounts_accept_the_service_name_prefix() { + let (client, _guard) = client(mount_external).await; + for path in ["/prpc/Worker.Version", "/prpc/v0/Worker.Version"] { + let (status, body) = get(&client, path).await; + assert_eq!(status, 200, "{path}: {body}"); + } + } + + /// The version in the path selects the surface, and nothing else does. + #[tokio::test] + async fn each_internal_mount_serves_only_its_own_surface() { + let (client, _guard) = client(mount_internal).await; + + // `Verify` is frozen-only; `IssueCert` is v1-only. + let (status, _) = get(&client, "/v1/Verify").await; + assert_ne!(status, 200, "/v1 must not serve the frozen Verify"); + + for path in ["/IssueCert", "/v0/IssueCert"] { + let (status, _) = get(&client, path).await; + assert_ne!(status, 200, "{path} must not serve the v1 IssueCert"); + } + } + + #[tokio::test] + async fn each_external_mount_serves_only_its_own_surface() { + let (client, _guard) = client(mount_external).await; + + // `Health` is v1-only on the external listener. + for path in ["/prpc/Health", "/prpc/v0/Health"] { + let (status, _) = get(&client, path).await; + assert_ne!(status, 200, "{path} must not serve the v1 Health"); + } + } + + /// How a client tells "this agent has no v1" from "v1 said no". + /// + /// Both an absent mount and an unknown method answer 404, so the status + /// alone is not enough: only the body separates them. A failed handler is + /// the 400. `docs/guest-api-v1.md` documents this as the probe rule, and + /// this test is what keeps the documented rule true. + #[tokio::test] + async fn version_probing_can_tell_an_absent_mount_from_an_unknown_method() { + let (state, _guard) = setup_test_state().await; + // An agent that predates v1: the frozen surface and nothing else. + let pre_v1 = Client::tracked( + rocket::build() + .mount("/", ra_rpc::prpc_routes!(AppState, InternalRpcHandler)) + .manage(state), + ) + .await + .expect("rocket failed to ignite"); + + // No `/v1` mount: Rocket has no route to match, and answers its own + // 404 page. This is what an agent too old for v1 looks like. + let (status, body) = get(&pre_v1, "/v1/GetKey").await; + assert_eq!(status, 404); + assert!(!body.contains("Service not found"), "{body}"); + + // A mounted surface, unknown method: also 404, but prpc's, naming the + // method. This is what a *current* agent says to a method it lacks. + let (status, body) = get(&pre_v1, "/NoSuchMethod").await; + assert_eq!(status, 404); + assert!(body.contains("Service not found: NoSuchMethod"), "{body}"); + + // A mounted surface, known method, handler says no: 400. + let (status, _) = get(&pre_v1, "/EmitEvent").await; + assert_eq!(status, 400); + } +} diff --git a/dstack/guest-agent/templates/dashboard.html b/dstack/guest-agent/templates/dashboard.html index 4cfd5494d..ee973bb90 100644 --- a/dstack/guest-agent/templates/dashboard.html +++ b/dstack/guest-agent/templates/dashboard.html @@ -21,9 +21,10 @@ body { font-family: 'Segoe UI', system-ui, -apple-system, sans-serif; - line-height: 1.6; - padding: 30px; - max-width: 1000px; + /* Data, not prose: 1.6 leading on single-line values reads as gaps. */ + line-height: 1.45; + padding: 24px 20px; + max-width: 1180px; margin: 0 auto; background-color: var(--background-color); color: var(--primary-color); @@ -32,9 +33,18 @@ h1, h2 { color: var(--primary-color); - margin-bottom: 1rem; border-bottom: 2px solid var(--border-color); - padding-bottom: 0.5rem; + padding-bottom: 0.3rem; + } + + h1 { + font-size: 1.6rem; + margin: 0 0 0.6rem; + } + + h2 { + font-size: 1.15rem; + margin: 1.4rem 0 0.6rem; } textarea { @@ -49,45 +59,43 @@ resize: vertical; } + /* Overflow belongs on a wrapper. Setting it on the table needs + display:block, which drops the table formatting context and lets + thead and tbody size themselves independently. */ + .table-scroll { + overflow-x: auto; + margin-top: 12px; + border-radius: 8px; + background-color: white; + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + } + table { width: 100%; border-collapse: separate; border-spacing: 0; - margin-top: 20px; - background-color: white; - border-radius: 8px; - box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + font-size: 0.92rem; } th, td { - padding: 15px; + padding: 7px 10px; text-align: left; border-bottom: 1px solid var(--border-color); } + /* An identifier split across two lines is harder to read than a table + the reader scrolls sideways. */ + td { + white-space: nowrap; + } + th { background-color: var(--primary-color); color: white; font-weight: 500; } - th:first-child { - border-top-left-radius: 8px; - } - - th:last-child { - border-top-right-radius: 8px; - } - - tr:last-child td:first-child { - border-bottom-left-radius: 8px; - } - - tr:last-child td:last-child { - border-bottom-right-radius: 8px; - } - tbody tr:hover { background-color: #f5f6f7; } @@ -104,27 +112,36 @@ background-color: #edf2f7; } + /* The global link padding gives the hover pill some body, but in a + table cell it also sets the row height -- once the cells themselves + are tight, the one row holding a link stands taller than the rest. */ + td a { + padding: 2px 8px; + margin-left: -8px; + } + .info-section { background-color: white; - padding: 20px; + padding: 6px 16px; border-radius: 8px; - margin-bottom: 20px; + margin-bottom: 14px; box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); } .info-grid { display: grid; - gap: 12px; - margin-top: 16px; } .info-row { display: grid; - grid-template-columns: 200px 1fr; + grid-template-columns: 190px 1fr; align-items: center; - padding: 4px 12px; - background-color: #f8f9fa; - border-radius: 6px; + padding: 5px 4px; + border-bottom: 1px solid var(--border-color); + } + + .info-row:last-child { + border-bottom: none; } .info-label { @@ -134,10 +151,6 @@ .info-value { font-family: monospace; - background-color: white; - padding: 6px 12px; - border-radius: 4px; - border: 1px solid var(--border-color); word-break: break-all; overflow-wrap: break-word; } @@ -197,11 +210,11 @@ <h1>Node Information</h1> </div> <div class="info-row"> <div class="info-label">Load Average</div> - <div class="info-value">1min: {{system_info.loadavg_one as f32 / 100.0}}%, 5min: {{system_info.loadavg_five as f32 / 100.0}}%, 15min: {{system_info.loadavg_fifteen as f32 / 100.0}}%</div> + <div class="info-value">1min: {{system_info.loadavg_one|load}}, 5min: {{system_info.loadavg_five|load}}, 15min: {{system_info.loadavg_fifteen|load}}</div> </div> <div class="info-row"> <div class="info-label">System Uptime</div> - <div class="info-value">{{system_info.uptime}}</div> + <div class="info-value">{{system_info.uptime|hduration}}</div> </div> {% for (i, disk) in system_info.disks.iter().enumerate() %} <div class="info-row"> @@ -213,33 +226,142 @@ <h1>Node Information</h1> </div> </div> + {% if public_sysinfo %} + <h2>GPUs</h2> + {% if !gpu_info.error.is_empty() %} + <div class="info-section">{{ gpu_info.error }}</div> + {% else if gpu_info.gpus.is_empty() %} + <div class="info-section">No NVIDIA GPUs</div> + {% else %} + <div class="info-section"> + <div class="info-grid"> + <div class="info-row"> + <div class="info-label">Confidential Computing</div> + <div class="info-value"> + {%- match gpu_info.cc_enabled -%} + {%- when Some with (enabled) -%} + {{- enabled -}} + {%- when None -%} + unknown + {%- endmatch -%} + </div> + </div> + <div class="info-row"> + <div class="info-label">GPU Ready State</div> + <div class="info-value"> + {%- match gpu_info.cc_ready -%} + {%- when Some with (ready) -%} + {{- ready -}} + {%- when None -%} + unknown + {%- endmatch -%} + </div> + </div> + {%- match gpu_info.sample_age_ms %} + {%- when Some with (age) %} + <div class="info-row"> + <div class="info-label">Sample Age</div> + <div class="info-value">{{ "{:.1}"|format(age as f32 / 1000.0) }} s</div> + </div> + {%- when None %} + {%- endmatch %} + </div> + </div> + <div class="table-scroll"> + <table> + <thead> + <tr> + <th>Index</th> + <th>PCI</th> + <th>GPU %</th> + <th>Mem %</th> + <th>Memory</th> + <th>Temp</th> + <th>Power</th> + <th>Error</th> + </tr> + </thead> + <tbody> + {% for gpu in gpu_info.gpus %} + <tr> + <td>{{ gpu.index }}</td> + <td>{{ gpu.pci_bus_id|short_bdf }}</td> + <td> + {% match gpu.utilization_gpu %} + {% when Some with (value) %}{{ value }}% + {% when None %}- + {% endmatch %} + </td> + <td> + {% match gpu.utilization_memory %} + {% when Some with (value) %}{{ value }}% + {% when None %}- + {% endmatch %} + </td> + <td> + {% match gpu.memory_used_bytes %} + {% when Some with (used) %} + {% match gpu.memory_total_bytes %} + {% when Some with (total) %}{{ used|hsize }} / {{ total|hsize }} + {% when None %}{{ used|hsize }} + {% endmatch %} + {% when None %} + {% match gpu.memory_total_bytes %} + {% when Some with (total) %}- / {{ total|hsize }} + {% when None %}- + {% endmatch %} + {% endmatch %} + </td> + <td> + {% match gpu.temperature_c %} + {% when Some with (value) %}{{ value }} C + {% when None %}- + {% endmatch %} + </td> + <td> + {% match gpu.power_usage_mw %} + {% when Some with (value) %}{{ "{:.1}"|format(value as f32 / 1000.0) }} W + {% when None %}- + {% endmatch %} + </td> + <td>{{ gpu.errors.join("; ") }}</td> + </tr> + {% endfor %} + </tbody> + </table> + </div> + {% endif %} + {% endif %} + <h2>Deployed Containers</h2> - <table> - <thead> - <tr> - <th>Name</th> - <th>Status</th> - {% if public_logs %} - <th>Logs</th> - {% endif %} - </tr> - </thead> - <tbody> - {% for container in containers %} - {% set name = container.names.get(0) %} - <tr> - <td>{{name|cname}}</td> - <td>{{container.status}}</td> - {% if public_logs %} - <td> - <a href="/logs/{{name|cname}}?text&bare×tamps&follow&tail=20" target="_blank">View - Logs</a> - </td> - {% endif %} - </tr> - {% endfor %} - </tbody> - </table> + <div class="table-scroll"> + <table> + <thead> + <tr> + <th>Name</th> + <th>Status</th> + {% if public_logs %} + <th>Logs</th> + {% endif %} + </tr> + </thead> + <tbody> + {% for container in containers %} + {% set name = container.names.get(0) %} + <tr> + <td>{{name|cname}}</td> + <td>{{container.status}}</td> + {% if public_logs %} + <td> + <a href="/logs/{{name|cname}}?text&bare×tamps&follow&tail=20" target="_blank">View + Logs</a> + </td> + {% endif %} + </tr> + {% endfor %} + </tbody> + </table> + </div> <h2>TCB Info</h2> {% if public_tcbinfo %} diff --git a/dstack/guest-agent/templates/metrics.tpl b/dstack/guest-agent/templates/metrics.tpl index 417bb9948..6c864c036 100644 --- a/dstack/guest-agent/templates/metrics.tpl +++ b/dstack/guest-agent/templates/metrics.tpl @@ -1,87 +1,240 @@ -# HELP system_os_name Operating system name +# HELP dstack_guest_info Static system information: OS, kernel and CPU model. +# TYPE dstack_guest_info gauge +dstack_guest_info{os_name="{{system_info.os_name|prometheus_label}}", os_version="{{system_info.os_version|prometheus_label}}", kernel_version="{{system_info.kernel_version|prometheus_label}}", cpu_model="{{system_info.cpu_model|prometheus_label}}"} 1 + +# HELP dstack_guest_cpus Number of logical CPUs. +# TYPE dstack_guest_cpus gauge +dstack_guest_cpus {{system_info.num_cpus}} + +# HELP dstack_guest_memory_total_bytes Total memory in bytes. +# TYPE dstack_guest_memory_total_bytes gauge +dstack_guest_memory_total_bytes {{system_info.total_memory}} + +# HELP dstack_guest_memory_available_bytes Available memory in bytes. +# TYPE dstack_guest_memory_available_bytes gauge +dstack_guest_memory_available_bytes {{system_info.available_memory}} + +# HELP dstack_guest_memory_used_bytes Used memory in bytes. +# TYPE dstack_guest_memory_used_bytes gauge +dstack_guest_memory_used_bytes {{system_info.used_memory}} + +# HELP dstack_guest_memory_free_bytes Free memory in bytes. +# TYPE dstack_guest_memory_free_bytes gauge +dstack_guest_memory_free_bytes {{system_info.free_memory}} + +# HELP dstack_guest_swap_total_bytes Total swap in bytes. +# TYPE dstack_guest_swap_total_bytes gauge +dstack_guest_swap_total_bytes {{system_info.total_swap}} + +# HELP dstack_guest_swap_used_bytes Used swap in bytes. +# TYPE dstack_guest_swap_used_bytes gauge +dstack_guest_swap_used_bytes {{system_info.used_swap}} + +# HELP dstack_guest_swap_free_bytes Free swap in bytes. +# TYPE dstack_guest_swap_free_bytes gauge +dstack_guest_swap_free_bytes {{system_info.free_swap}} + +# HELP dstack_guest_uptime_seconds System uptime in seconds. +# TYPE dstack_guest_uptime_seconds gauge +dstack_guest_uptime_seconds {{system_info.uptime}} + +# HELP dstack_guest_load1 System load average over 1 minute. +# TYPE dstack_guest_load1 gauge +dstack_guest_load1 {{system_info.loadavg_one|load}} + +# HELP dstack_guest_load5 System load average over 5 minutes. +# TYPE dstack_guest_load5 gauge +dstack_guest_load5 {{system_info.loadavg_five|load}} + +# HELP dstack_guest_load15 System load average over 15 minutes. +# TYPE dstack_guest_load15 gauge +dstack_guest_load15 {{system_info.loadavg_fifteen|load}} + +# HELP dstack_guest_disk_total_bytes Disk size in bytes. +# TYPE dstack_guest_disk_total_bytes gauge +{% for disk in system_info.disks %} +dstack_guest_disk_total_bytes{name="{{disk.name|prometheus_label}}", mount_point="{{disk.mount_point|prometheus_label}}"} {{disk.total_size}} +{% endfor %} + +# HELP dstack_guest_disk_free_bytes Free disk space in bytes. +# TYPE dstack_guest_disk_free_bytes gauge +{% for disk in system_info.disks %} +dstack_guest_disk_free_bytes{name="{{disk.name|prometheus_label}}", mount_point="{{disk.mount_point|prometheus_label}}"} {{disk.free_size}} +{% endfor %} + +# HELP dstack_guest_disk_used_bytes Used disk space in bytes. +# TYPE dstack_guest_disk_used_bytes gauge +{% for disk in system_info.disks %} +dstack_guest_disk_used_bytes{name="{{disk.name|prometheus_label}}", mount_point="{{disk.mount_point|prometheus_label}}"} {{disk.total_size - disk.free_size}} +{% endfor %} + +# HELP dstack_guest_disk_used_ratio Used fraction of the disk, 0 to 1. +# TYPE dstack_guest_disk_used_ratio gauge +{% for disk in system_info.disks %} +dstack_guest_disk_used_ratio{name="{{disk.name|prometheus_label}}", mount_point="{{disk.mount_point|prometheus_label}}"} {% if disk.total_size > 0 %}{{(disk.total_size - disk.free_size) as f64 / disk.total_size as f64}}{% else %}0{% endif %} +{% endfor %} + +# HELP dstack_gpu_nvml_up 1 if the last GPU sample succeeded, 0 otherwise. +# TYPE dstack_gpu_nvml_up gauge +dstack_gpu_nvml_up {% if gpu_info.error.is_empty() %}1{% else %}0{% endif %} + +{%- match gpu_info.sample_age_ms %}{% when Some with (age) %} + +# HELP dstack_gpu_sample_age_seconds Age of the GPU sample being served. +# TYPE dstack_gpu_sample_age_seconds gauge +dstack_gpu_sample_age_seconds {{ age as f64 / 1000.0 }} +{%- when None %}{% endmatch %} + +{%- match gpu_info.cc_enabled %}{% when Some with (enabled) %} + +# HELP dstack_gpu_cc_enabled 1 if NVIDIA confidential computing is enabled. +# TYPE dstack_gpu_cc_enabled gauge +dstack_gpu_cc_enabled {% if enabled %}1{% else %}0{% endif %} +{%- when None %}{% endmatch %} + +{%- match gpu_info.cc_ready %}{% when Some with (ready) %} + +# HELP dstack_gpu_cc_ready 1 if NVIDIA CC GPUs are accepting client requests. +# TYPE dstack_gpu_cc_ready gauge +dstack_gpu_cc_ready {% if ready %}1{% else %}0{% endif %} +{%- when None %}{% endmatch %} + +# HELP dstack_gpu_utilization_percent GPU SM duty cycle, percent. +# TYPE dstack_gpu_utilization_percent gauge +{%- for gpu in gpu_info.gpus %}{% match gpu.utilization_gpu %}{% when Some with (value) %} +dstack_gpu_utilization_percent{{ gpu|gpu_labels }} {{ value }} +{%- when None %}{% endmatch %}{% endfor %} + +# HELP dstack_gpu_memory_utilization_percent GPU memory-bus duty cycle, percent. +# TYPE dstack_gpu_memory_utilization_percent gauge +{%- for gpu in gpu_info.gpus %}{% match gpu.utilization_memory %}{% when Some with (value) %} +dstack_gpu_memory_utilization_percent{{ gpu|gpu_labels }} {{ value }} +{%- when None %}{% endmatch %}{% endfor %} + +# HELP dstack_gpu_memory_total_bytes GPU framebuffer size in bytes. +# TYPE dstack_gpu_memory_total_bytes gauge +{%- for gpu in gpu_info.gpus %}{% match gpu.memory_total_bytes %}{% when Some with (value) %} +dstack_gpu_memory_total_bytes{{ gpu|gpu_labels }} {{ value }} +{%- when None %}{% endmatch %}{% endfor %} + +# HELP dstack_gpu_memory_used_bytes GPU framebuffer used in bytes. +# TYPE dstack_gpu_memory_used_bytes gauge +{%- for gpu in gpu_info.gpus %}{% match gpu.memory_used_bytes %}{% when Some with (value) %} +dstack_gpu_memory_used_bytes{{ gpu|gpu_labels }} {{ value }} +{%- when None %}{% endmatch %}{% endfor %} + +# HELP dstack_gpu_memory_free_bytes GPU framebuffer free in bytes. +# TYPE dstack_gpu_memory_free_bytes gauge +{%- for gpu in gpu_info.gpus %}{% match gpu.memory_free_bytes %}{% when Some with (value) %} +dstack_gpu_memory_free_bytes{{ gpu|gpu_labels }} {{ value }} +{%- when None %}{% endmatch %}{% endfor %} + +# HELP dstack_gpu_temperature_celsius GPU temperature in Celsius. +# TYPE dstack_gpu_temperature_celsius gauge +{%- for gpu in gpu_info.gpus %}{% match gpu.temperature_c %}{% when Some with (value) %} +dstack_gpu_temperature_celsius{{ gpu|gpu_labels }} {{ value }} +{%- when None %}{% endmatch %}{% endfor %} + +# HELP dstack_gpu_power_usage_milliwatts GPU power usage in milliwatts. +# TYPE dstack_gpu_power_usage_milliwatts gauge +{%- for gpu in gpu_info.gpus %}{% match gpu.power_usage_mw %}{% when Some with (value) %} +dstack_gpu_power_usage_milliwatts{{ gpu|gpu_labels }} {{ value }} +{%- when None %}{% endmatch %}{% endfor %} + +# HELP dstack_gpu_query_errors Number of NVML field queries that failed on this GPU. +# TYPE dstack_gpu_query_errors gauge +{%- for gpu in gpu_info.gpus %} +dstack_gpu_query_errors{{ gpu|gpu_labels }} {{ gpu.errors.len() }} +{%- endfor %} + +# Everything below is the pre-rename exposition, kept verbatim so existing +# dashboards keep working through one release cycle. Deprecated: use the +# dstack_guest_* series above; these will be removed in a future release. + +# HELP system_os_name Operating system name (deprecated: use dstack_guest_info) # TYPE system_os_name gauge -system_os_name{os_name="{{system_info.os_name}}"} 1 +system_os_name{os_name="{{system_info.os_name|prometheus_label}}"} 1 -# HELP system_os_version Operating system version +# HELP system_os_version Operating system version (deprecated: use dstack_guest_info) # TYPE system_os_version gauge -system_os_version{os_version="{{system_info.os_version}}"} 1 +system_os_version{os_version="{{system_info.os_version|prometheus_label}}"} 1 -# HELP system_kernel_version Kernel version +# HELP system_kernel_version Kernel version (deprecated: use dstack_guest_info) # TYPE system_kernel_version gauge -system_kernel_version{kernel_version="{{system_info.kernel_version}}"} 1 +system_kernel_version{kernel_version="{{system_info.kernel_version|prometheus_label}}"} 1 -# HELP system_cpu_model CPU model information +# HELP system_cpu_model CPU model information (deprecated: use dstack_guest_info) # TYPE system_cpu_model gauge -system_cpu_model{cpu_model="{{system_info.cpu_model}}"} 1 +system_cpu_model{cpu_model="{{system_info.cpu_model|prometheus_label}}"} 1 -# HELP system_num_cpus Number of logical CPUs +# HELP system_num_cpus Number of logical CPUs (deprecated: use dstack_guest_cpus) # TYPE system_num_cpus gauge system_num_cpus {{system_info.num_cpus}} -# HELP system_memory_total Total memory in bytes +# HELP system_memory_total Total memory in bytes (deprecated: use dstack_guest_memory_total_bytes) # TYPE system_memory_total gauge system_memory_total {{system_info.total_memory}} -# HELP system_memory_available Available memory in bytes +# HELP system_memory_available Available memory in bytes (deprecated: use dstack_guest_memory_available_bytes) # TYPE system_memory_available gauge system_memory_available {{system_info.available_memory}} -# HELP system_memory_used Used memory in bytes +# HELP system_memory_used Used memory in bytes (deprecated: use dstack_guest_memory_used_bytes) # TYPE system_memory_used gauge system_memory_used {{system_info.used_memory}} -# HELP system_memory_free Free memory in bytes +# HELP system_memory_free Free memory in bytes (deprecated: use dstack_guest_memory_free_bytes) # TYPE system_memory_free gauge system_memory_free {{system_info.free_memory}} -# HELP system_swap_total Total swap memory in bytes +# HELP system_swap_total Total swap memory in bytes (deprecated: use dstack_guest_swap_total_bytes) # TYPE system_swap_total gauge system_swap_total {{system_info.total_swap}} -# HELP system_swap_used Used swap memory in bytes +# HELP system_swap_used Used swap memory in bytes (deprecated: use dstack_guest_swap_used_bytes) # TYPE system_swap_used gauge system_swap_used {{system_info.used_swap}} -# HELP system_swap_free Free swap memory in bytes +# HELP system_swap_free Free swap memory in bytes (deprecated: use dstack_guest_swap_free_bytes) # TYPE system_swap_free gauge system_swap_free {{system_info.free_swap}} -# HELP system_uptime System uptime in seconds +# HELP system_uptime System uptime in seconds (deprecated: use dstack_guest_uptime_seconds) # TYPE system_uptime gauge system_uptime {{system_info.uptime}} -# HELP system_load_average_1m System load average (1 minute) +# HELP system_load_average_1m System load average (1 minute) (deprecated: use dstack_guest_load1) # TYPE system_load_average_1m gauge -system_load_average_1m {{system_info.loadavg_one}} +system_load_average_1m {{system_info.loadavg_one|load}} -# HELP system_load_average_5m System load average (5 minutes) +# HELP system_load_average_5m System load average (5 minutes) (deprecated: use dstack_guest_load5) # TYPE system_load_average_5m gauge -system_load_average_5m {{system_info.loadavg_five}} +system_load_average_5m {{system_info.loadavg_five|load}} -# HELP system_load_average_15m System load average (15 minutes) +# HELP system_load_average_15m System load average (15 minutes) (deprecated: use dstack_guest_load15) # TYPE system_load_average_15m gauge -system_load_average_15m {{system_info.loadavg_fifteen}} +system_load_average_15m {{system_info.loadavg_fifteen|load}} -# HELP disk_total_size Disk total size in bytes +# HELP disk_total_size Disk total size in bytes (deprecated: use dstack_guest_disk_total_bytes) # TYPE disk_total_size gauge {% for disk in system_info.disks %} -disk_total_size{name="{{disk.name}}", mount_point="{{disk.mount_point}}"} {{disk.total_size}} +disk_total_size{name="{{disk.name|prometheus_label}}", mount_point="{{disk.mount_point|prometheus_label}}"} {{disk.total_size}} {% endfor %} -# HELP disk_free_size Disk free size in bytes +# HELP disk_free_size Disk free size in bytes (deprecated: use dstack_guest_disk_free_bytes) # TYPE disk_free_size gauge {% for disk in system_info.disks %} -disk_free_size{name="{{disk.name}}", mount_point="{{disk.mount_point}}"} {{disk.free_size}} +disk_free_size{name="{{disk.name|prometheus_label}}", mount_point="{{disk.mount_point|prometheus_label}}"} {{disk.free_size}} {% endfor %} -# HELP disk_used_size Disk used size in bytes +# HELP disk_used_size Disk used size in bytes (deprecated: use dstack_guest_disk_used_bytes) # TYPE disk_used_size gauge {% for disk in system_info.disks %} -disk_used_size{name="{{disk.name}}", mount_point="{{disk.mount_point}}"} {{disk.total_size - disk.free_size}} +disk_used_size{name="{{disk.name|prometheus_label}}", mount_point="{{disk.mount_point|prometheus_label}}"} {{disk.total_size - disk.free_size}} {% endfor %} -# HELP disk_usage_percentage Disk usage percentage +# HELP disk_usage_percentage Disk usage percentage (deprecated: use dstack_guest_disk_used_ratio) # TYPE disk_usage_percentage gauge {% for disk in system_info.disks %} -disk_usage_percentage{name="{{disk.name}}", mount_point="{{disk.mount_point}}"} {% if disk.total_size > 0 %}{{(disk.total_size - disk.free_size) as f64 / disk.total_size as f64 * 100.0}}{% else %}0{% endif %} +disk_usage_percentage{name="{{disk.name|prometheus_label}}", mount_point="{{disk.mount_point|prometheus_label}}"} {% if disk.total_size > 0 %}{{(disk.total_size - disk.free_size) as f64 / disk.total_size as f64 * 100.0}}{% else %}0{% endif %} {% endfor %} diff --git a/dstack/guest-api/proto/guest_api.proto b/dstack/guest-api/proto/guest_api.proto index 5d5868cb9..9e2a9f990 100644 --- a/dstack/guest-api/proto/guest_api.proto +++ b/dstack/guest-api/proto/guest_api.proto @@ -129,12 +129,70 @@ message DiskInfo { uint64 free_size = 5; } +// One NVIDIA GPU sampled through NVML. +// +// Optional numeric fields are unset when that query failed, so a consumer can +// tell a genuine zero from a missing sample. `errors` lists the failed queries +// for this card, one entry each, so counting them does not mean re-parsing a +// joined string. +// +// `uuid` is the stable identity; `index` and `pci_bus_id` are ordinal/location +// helpers. `uuid` and `pci_bus_id` are plain strings rather than `optional` +// because an empty value is already unambiguous: NVML never returns one, and +// the matching `errors` entry names the query that failed. `index` is always +// populated, so a device is identifiable even when both lookups fail. +message GpuDevice { + // NVML device index (PCI order) + uint32 index = 1; + string uuid = 2; + // PCI BDF from nvmlPciInfo.busId + string pci_bus_id = 3; + // SM duty cycle, percent + optional uint32 utilization_gpu = 4; + // Memory-bus duty cycle, percent + optional uint32 utilization_memory = 5; + optional uint64 memory_total_bytes = 6; + optional uint64 memory_used_bytes = 7; + optional uint64 memory_free_bytes = 8; + optional uint32 temperature_c = 9; + optional uint32 power_usage_mw = 10; + // Failed queries for this card, one per entry; empty if all fields succeeded + repeated string errors = 11; +} + +// GPU inventory collected through NVML. +// +// `error` is non-empty when NVML itself is unavailable, or when the sample +// could not be taken at all. In that case `gpus` is empty. An empty `gpus` +// with an empty `error` means the collector ran and the guest has no NVIDIA +// GPUs. +// +// `cc_ready` and `cc_enabled` are both system-wide, not per device: NVML +// exposes them as `nvmlSystemGetConfComputeGpusReadyState` and +// `nvmlSystemGetConfComputeSettings`, which nvml-wrapper hangs off `Device` +// without using the device handle. `cc_enabled` says the CC feature is turned +// on; `cc_ready` says the SPDM handshake finished and the GPUs are accepting +// client work. Both unset when the query is unsupported or failed. +// +// `sample_age_ms` is how old the returned sample is. The agent serves the last +// known snapshot rather than nothing, so consumers must decide for themselves +// how stale is too stale. Unset when the response carries no sample. +message GpuInfoResponse { + repeated GpuDevice gpus = 1; + string error = 2; + optional bool cc_ready = 3; + optional bool cc_enabled = 4; + optional uint64 sample_age_ms = 5; +} + // Direct gRPC surface exposed by the in-guest agent. service GuestApi { // Returns attestation material and identifiers for the calling guest. rpc Info(google.protobuf.Empty) returns (GuestInfo); // Reports the guest's OS/kernel and resource statistics. rpc SysInfo(google.protobuf.Empty) returns (SystemInfo); + // Reports per-GPU utilization, memory, temperature, power, and CC ready state. + rpc GpuInfo(google.protobuf.Empty) returns (GpuInfoResponse); // Dumps NIC/Gateway configuration so operators can debug connectivity. rpc NetworkInfo(google.protobuf.Empty) returns (NetworkInformation); // Enumerates the containers running under the guest supervisor. @@ -147,6 +205,7 @@ service GuestApi { service ProxiedGuestApi { rpc Info(Id) returns (GuestInfo); rpc SysInfo(Id) returns (SystemInfo); + rpc GpuInfo(Id) returns (GpuInfoResponse); rpc NetworkInfo(Id) returns (NetworkInformation); rpc ListContainers(Id) returns (ListContainersResponse); rpc Shutdown(Id) returns (google.protobuf.Empty); diff --git a/dstack/guest-api/src/client.rs b/dstack/guest-api/src/client.rs index 68fa36e44..d73a22df2 100644 --- a/dstack/guest-api/src/client.rs +++ b/dstack/guest-api/src/client.rs @@ -4,9 +4,44 @@ use crate::guest_api_client::GuestApiClient; use http_client::prpc::PrpcClient; +use std::time::Duration; pub type DefaultClient = GuestApiClient<PrpcClient>; +/// Largest response the host will read from a guest agent. +/// +/// The peer is a tenant's CVM, which the host does not trust: without a bound, +/// one guest answering `SysInfo` or `ListContainers` with an endless body takes +/// the VMM down, and the VMM is the control plane for every other tenant on the +/// machine. Sized well above anything these methods legitimately return. +pub const MAX_RESPONSE_BYTES: usize = 16 * 1024 * 1024; + pub fn new_client(base_url: String) -> DefaultClient { - DefaultClient::new(PrpcClient::new(base_url)) + DefaultClient::new(PrpcClient::new(base_url).with_max_response_bytes(MAX_RESPONSE_BYTES)) +} + +/// Both guards, because they defend against the same untrusted peer: the +/// timeout covers a guest that answers slowly, the bound covers one that +/// answers endlessly, and a caller wanting one has no reason to drop the other. +pub fn new_client_with_timeout(base_url: String, timeout: Duration) -> DefaultClient { + DefaultClient::new( + PrpcClient::new(base_url) + .with_max_response_bytes(MAX_RESPONSE_BYTES) + .with_request_timeout(timeout), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The VMM reaches a tenant's guest agent through the timeout constructor, + /// so a bound that only the plain one applies protects nothing. + #[test] + fn the_timeout_client_is_still_bounded() { + let client = PrpcClient::new("vsock://3:8000/api".to_string()) + .with_max_response_bytes(MAX_RESPONSE_BYTES) + .with_request_timeout(Duration::from_secs(1)); + assert_eq!(client.max_response_bytes(), Some(MAX_RESPONSE_BYTES)); + } } diff --git a/dstack/guest-api/src/lib.rs b/dstack/guest-api/src/lib.rs index eb5524134..36aae1ad1 100644 --- a/dstack/guest-api/src/lib.rs +++ b/dstack/guest-api/src/lib.rs @@ -10,3 +10,92 @@ mod generated; #[cfg(feature = "client")] pub mod client; + +#[cfg(test)] +mod tests { + use super::*; + use prost::Message; + + /// Every numeric field is `optional` so a consumer can tell "the query + /// failed" from a genuine zero. That distinction only holds if it survives + /// the wire, which is what this pins. + #[test] + fn gpu_info_response_roundtrips_optional_fields() { + let original = GpuInfoResponse { + error: String::new(), + cc_ready: Some(true), + cc_enabled: Some(true), + sample_age_ms: Some(4200), + gpus: vec![GpuDevice { + index: 0, + uuid: "GPU-abc".into(), + pci_bus_id: "00000000:01:00.0".into(), + utilization_gpu: Some(12), + utilization_memory: None, + memory_total_bytes: Some(8 << 30), + memory_used_bytes: Some(0), + memory_free_bytes: None, + temperature_c: Some(0), + power_usage_mw: None, + errors: vec!["temperature: timeout".into()], + }], + }; + let bytes = original.encode_to_vec(); + let decoded = GpuInfoResponse::decode(bytes.as_slice()).expect("decode"); + assert_eq!(decoded, original); + assert_eq!(decoded.gpus[0].utilization_memory, None); + assert_eq!(decoded.gpus[0].memory_used_bytes, Some(0)); + assert_eq!(decoded.cc_ready, Some(true)); + assert_eq!(decoded.sample_age_ms, Some(4200)); + } + + /// The NVML-unavailable shape: an error, no devices, and no CC state at + /// all. `cc_ready` unset must not decode as `Some(false)`, which would + /// read as "the handshake failed" rather than "we could not ask". + #[test] + fn gpu_info_response_omits_unset_optionals() { + let original = GpuInfoResponse { + error: "failed to initialize NVML: driver not loaded".into(), + cc_ready: None, + cc_enabled: None, + sample_age_ms: None, + gpus: vec![], + }; + let decoded = GpuInfoResponse::decode(original.encode_to_vec().as_slice()).expect("decode"); + assert_eq!(decoded, original); + assert!(decoded.gpus.is_empty()); + assert!(!decoded.error.is_empty()); + assert_eq!(decoded.cc_ready, None); + assert_eq!(decoded.cc_enabled, None); + } + + /// "No GPUs" is the default response: empty devices, empty error. It must + /// stay distinguishable from every failure shape above. + #[test] + fn the_default_response_means_no_gpus_rather_than_a_failure() { + let decoded = + GpuInfoResponse::decode(GpuInfoResponse::default().encode_to_vec().as_slice()) + .expect("decode"); + assert!(decoded.gpus.is_empty()); + assert!(decoded.error.is_empty()); + assert_eq!(decoded.sample_age_ms, None); + } + + /// Per-field failures are a list, so counting them for + /// `dstack_gpu_query_errors` does not mean re-parsing a joined string. + #[test] + fn per_device_errors_are_counted_not_parsed() { + let device = GpuDevice { + index: 0, + errors: vec![ + "power: not supported".into(), + // A message containing the old "; " separator would have + // inflated the count when errors were a joined string. + "memory: unknown error; retry advised".into(), + ], + ..Default::default() + }; + let decoded = GpuDevice::decode(device.encode_to_vec().as_slice()).expect("decode"); + assert_eq!(decoded.errors.len(), 2); + } +} diff --git a/dstack/http-client/Cargo.toml b/dstack/http-client/Cargo.toml index fffe277ab..d8aea3fbd 100644 --- a/dstack/http-client/Cargo.toml +++ b/dstack/http-client/Cargo.toml @@ -25,6 +25,7 @@ tokio-vsock.workspace = true tower-service = "0.3.3" [dev-dependencies] +prost.workspace = true tokio = { workspace = true, features = ["full"] } [features] diff --git a/dstack/http-client/src/hyper_vsock.rs b/dstack/http-client/src/hyper_vsock.rs index df280f80b..a9104ee23 100644 --- a/dstack/http-client/src/hyper_vsock.rs +++ b/dstack/http-client/src/hyper_vsock.rs @@ -2,13 +2,10 @@ // // SPDX-License-Identifier: Apache-2.0 -use hyper::{body::Body, rt::ReadBufCursor, Uri}; +use hyper::{rt::ReadBufCursor, Uri}; use hyper_util::{ - client::legacy::{ - connect::{Connected, Connection}, - Client, - }, - rt::{TokioExecutor, TokioIo}, + client::legacy::connect::{Connected, Connection}, + rt::TokioIo, }; use pin_project_lite::pin_project; use std::{ @@ -169,16 +166,4 @@ fn parse_vsock_host(uri: &Uri) -> Result<(u32, u32), io::Error> { Ok((cid, port)) } -/// Extension trait for constructing a hyper HTTP client over a Vsock -pub trait VsockClientExt<B: Body + Send> { - /// Construct a client which speaks HTTP over a Vsock domain socket - #[must_use] - fn vsock() -> Client<VsockConnector, B> - where - B::Data: Send, - { - Client::builder(TokioExecutor::new()).build(VsockConnector) - } -} - -impl<B: Body + Send> VsockClientExt<B> for Client<VsockConnector, B> {} +// The client itself is built in `lib.rs`, once per connection-reuse policy. diff --git a/dstack/http-client/src/lib.rs b/dstack/http-client/src/lib.rs index e33d89ae1..5b41d09af 100644 --- a/dstack/http-client/src/lib.rs +++ b/dstack/http-client/src/lib.rs @@ -2,20 +2,144 @@ // // SPDX-License-Identifier: Apache-2.0 -use anyhow::Result; +use anyhow::{anyhow, bail, Result}; use http_body_util::{BodyExt, Full}; use hyper::body::Bytes; use hyper::Request; use hyper_util::client::legacy::Client; -use hyper_vsock::VsockClientExt; -use hyperlocal::{UnixClientExt, UnixConnector, Uri}; +use hyper_util::rt::TokioExecutor; +use hyper_vsock::VsockConnector; +use hyperlocal::{UnixConnector, Uri}; use log::debug; +use std::sync::OnceLock; mod hyper_vsock; #[cfg(feature = "prpc")] pub mod prpc; +/// A response bound for callers whose peer is not trusted. +/// +/// Not a default. Most users of this transport talk to something local or +/// operator-run and legitimately fetch as much as they asked for -- `dstack vmm +/// logs --lines 20000` is a megabyte on its own. But some talk to a CVM's guest +/// agent, which is exactly the party dstack's threat model does not trust, and +/// at least one of those polls on a timer against the whole fleet. Those callers +/// pass this. Compare `gateway/src/kv/https_client.rs`, which learned the same +/// lesson on the peer-facing sync client. +/// +/// Far above anything a control-plane RPC returns, and far below anything that +/// matters. +pub const MAX_RESPONSE_BYTES: usize = 1024 * 1024; + +/// Whether a request may travel over a connection an earlier one left open. +/// +/// The default is to reuse. A control-plane call is strictly cheaper over a +/// warm connection, and a connection that died while idle costs one failed +/// request, which the caller was going to have to handle anyway. +/// +/// A health probe is the exception, and the reason this is a decision a caller +/// states rather than a setting it inherits. Reuse there can report healthy +/// over a connection that real traffic would never be given: an agent out of +/// file descriptors, or with a full accept backlog, keeps serving whoever is +/// already connected while refusing everyone new. Every connection the gateway +/// proxies to an app is a new one, so "can a connection be opened and answered +/// right now" is the question the probe exists to ask -- and a pooled +/// connection is not asking it. Failing the other way is bounded by +/// comparison: a probe that trips over a dead pooled connection reports +/// unreachable, which needs `failure_threshold` of them in a row to become a +/// verdict, and the pool drops the connection on the way out. +/// +/// This is not a new position for the project. `tappd`'s own watchdog probe +/// stopped reusing connections in fb0688b7f0 (#140) -- by moving +/// `reqwest::Client::new()` inside the loop, which is the same conclusion +/// reached without a way to say so. `Fresh` is that decision with a name, and +/// without the per-probe client construction it had to pay for it. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ConnectionReuse { + /// Keep connections alive between requests to the same host. + #[default] + Pooled, + /// Open a connection for this request and do not keep it. + Fresh, +} + +/// How to make a request, beyond where to send it. +/// +/// A struct rather than two more positional arguments: this list has grown +/// once already and every entry is the kind that reads as an unlabelled `None` +/// or `true` at the call site. Construct with `..Default::default()` so a +/// field added later does not break callers. +#[derive(Clone, Copy, Debug, Default)] +pub struct RequestOptions { + /// Refuse a response body larger than this. `None` accumulates whatever + /// the peer sends; see [`MAX_RESPONSE_BYTES`]. + pub max_response_bytes: Option<usize>, + /// See [`ConnectionReuse`]. + pub connection_reuse: ConnectionReuse, +} + +/// The HTTP client every request over `base` shares, one per reuse policy. +/// +/// Built once and kept. A `reqwest::Client` is a handle around a connection +/// pool, a DNS resolver and a TLS configuration; building one per request -- +/// which this module did until now -- pays for all three every time and then +/// discards the pool before anything can use it. Sharing the handle is what +/// removes that cost; sharing *connections* is a separate decision, which is +/// why `Fresh` is a second client rather than a flag on a request. +fn tcp_client(reuse: ConnectionReuse) -> Result<&'static reqwest::Client> { + static POOLED: OnceLock<std::result::Result<reqwest::Client, String>> = OnceLock::new(); + static FRESH: OnceLock<std::result::Result<reqwest::Client, String>> = OnceLock::new(); + let cell = match reuse { + ConnectionReuse::Pooled => &POOLED, + ConnectionReuse::Fresh => &FRESH, + }; + cell.get_or_init(|| { + let builder = match reuse { + ConnectionReuse::Pooled => reqwest::Client::builder(), + // Zero, not a short idle timeout: the pool never hands a + // connection back, so every request dials. A timeout would leave a + // window in which it still does. + ConnectionReuse::Fresh => reqwest::Client::builder().pool_max_idle_per_host(0), + }; + builder.build().map_err(|err| err.to_string()) + }) + .as_ref() + .map_err(|err| anyhow!("failed to build the HTTP client: {err}")) +} + +/// As [`tcp_client`], for the Unix-socket transport. +fn unix_client(reuse: ConnectionReuse) -> &'static Client<UnixConnector, Full<Bytes>> { + static POOLED: OnceLock<Client<UnixConnector, Full<Bytes>>> = OnceLock::new(); + static FRESH: OnceLock<Client<UnixConnector, Full<Bytes>>> = OnceLock::new(); + match reuse { + ConnectionReuse::Pooled => { + POOLED.get_or_init(|| Client::builder(TokioExecutor::new()).build(UnixConnector)) + } + ConnectionReuse::Fresh => FRESH.get_or_init(|| { + Client::builder(TokioExecutor::new()) + .pool_max_idle_per_host(0) + .build(UnixConnector) + }), + } +} + +/// As [`tcp_client`], for the vsock transport. +fn vsock_client(reuse: ConnectionReuse) -> &'static Client<VsockConnector, Full<Bytes>> { + static POOLED: OnceLock<Client<VsockConnector, Full<Bytes>>> = OnceLock::new(); + static FRESH: OnceLock<Client<VsockConnector, Full<Bytes>>> = OnceLock::new(); + match reuse { + ConnectionReuse::Pooled => { + POOLED.get_or_init(|| Client::builder(TokioExecutor::new()).build(VsockConnector)) + } + ConnectionReuse::Fresh => FRESH.get_or_init(|| { + Client::builder(TokioExecutor::new()) + .pool_max_idle_per_host(0) + .build(VsockConnector) + }), + } +} + fn mk_url(base: &str, path: &str) -> String { let base = base.trim_end_matches('/'); let path = path.trim_start_matches('/'); @@ -35,6 +159,63 @@ pub async fn http_request( path: &str, body: &[u8], ) -> Result<(u16, Vec<u8>)> { + http_request_with_headers(method, base, path, body, &[]).await +} + +/// Same as [`http_request`], with extra request headers (e.g. `Authorization`). +pub async fn http_request_with_headers( + method: &str, + base: &str, + path: &str, + body: &[u8], + headers: &[(&str, &str)], +) -> Result<(u16, Vec<u8>)> { + http_request_bounded(method, base, path, body, headers, None).await +} + +/// Same as [`http_request_with_headers`], refusing a response larger than +/// `max_bytes`. +/// +/// `None` accumulates whatever the peer sends, which is what a caller fetching +/// logs or a large listing wants. Callers whose peer is untrusted -- anything +/// talking to a CVM's guest agent -- should pass [`MAX_RESPONSE_BYTES`]; see the +/// note there. +pub async fn http_request_bounded( + method: &str, + base: &str, + path: &str, + body: &[u8], + headers: &[(&str, &str)], + max_bytes: Option<usize>, +) -> Result<(u16, Vec<u8>)> { + http_request_with_options( + method, + base, + path, + body, + headers, + RequestOptions { + max_response_bytes: max_bytes, + ..Default::default() + }, + ) + .await +} + +/// Same as [`http_request_bounded`], also choosing whether the request may +/// reuse a connection. See [`ConnectionReuse`]. +pub async fn http_request_with_options( + method: &str, + base: &str, + path: &str, + body: &[u8], + headers: &[(&str, &str)], + options: RequestOptions, +) -> Result<(u16, Vec<u8>)> { + let RequestOptions { + max_response_bytes: max_bytes, + connection_reuse, + } = options; debug!("Sending HTTP request to {base}, path={path}"); let mut response = if let Some(uds) = base.strip_prefix("unix:") { let path = if path.starts_with("/") { @@ -42,50 +223,70 @@ pub async fn http_request( } else { format!("/{path}") }; - let client: Client<UnixConnector, Full<Bytes>> = Client::unix(); + let client = unix_client(connection_reuse); let unix_uri: hyper::Uri = Uri::new(uds, &path).into(); - let req = Request::builder() - .method(method) - .uri(unix_uri) - .body(Full::new(Bytes::copy_from_slice(body)))?; + let mut builder = Request::builder().method(method).uri(unix_uri); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + let req = builder.body(Full::new(Bytes::copy_from_slice(body)))?; client.request(req).await? } else if base.starts_with("vsock:") { - let client = Client::vsock(); + let client = vsock_client(connection_reuse); let uri = mk_url(base, path).parse::<hyper::Uri>()?; - let req = Request::builder() - .method(method) - .uri(uri) - .body(Full::new(Bytes::copy_from_slice(body)))?; + let mut builder = Request::builder().method(method).uri(uri); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + let req = builder.body(Full::new(Bytes::copy_from_slice(body)))?; client.request(req).await? } else { let uri = mk_url(base, path); - let client = reqwest::Client::builder().build()?; + let client = tcp_client(connection_reuse)?; let method = reqwest::Method::from_bytes(method.as_bytes())?; let mut request = client.request(method, uri); + for (name, value) in headers { + request = request.header(*name, *value); + } if !body.is_empty() { request = request.body(body.to_vec()); } - let response = request.send().await?; - return Ok(( - response.status().as_u16(), - response.text().await?.into_bytes(), - )); + let mut response = request.send().await?; + let status = response.status().as_u16(); + let mut body = Vec::new(); + // Chunk by chunk rather than `text()`, so the bound is enforced while + // the body is still arriving instead of after it has been buffered. + while let Some(chunk) = response.chunk().await? { + push_bounded(&mut body, &chunk, max_bytes)?; + } + return Ok((status, body)); }; debug!("Response: {:?}", response); let mut body = Vec::new(); while let Some(frame_result) = response.frame().await { let frame = frame_result?; if let Some(segment) = frame.data_ref() { - body.extend_from_slice(segment.iter().as_slice()); + push_bounded(&mut body, segment.iter().as_slice(), max_bytes)?; } } Ok((response.status().as_u16(), body)) } +fn push_bounded(body: &mut Vec<u8>, segment: &[u8], max_bytes: Option<usize>) -> Result<()> { + if let Some(max) = max_bytes { + if body.len() + segment.len() > max { + bail!("response body exceeds {max} bytes"); + } + } + body.extend_from_slice(segment); + Ok(()) +} + #[cfg(test)] mod tests { use super::*; use std::error::Error; + use std::sync::atomic::{AtomicUsize, Ordering}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; @@ -124,4 +325,252 @@ mod tests { ); Ok(()) } + + /// Several callers of this transport talk to peers they do not control -- + /// a CVM's guest agent answering `Worker.Health` or `Info` -- and at least + /// one of them does it on a timer against the whole fleet. An unbounded + /// body there is an out-of-memory the peer gets to schedule. + #[tokio::test] + async fn an_oversized_response_body_is_refused() -> Result<(), Box<dyn Error>> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let oversized = MAX_RESPONSE_BYTES + 1; + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + let mut buf = [0u8; 1024]; + let _ = socket.read(&mut buf).await?; + socket + .write_all( + format!("HTTP/1.1 200 OK\r\nContent-Length: {oversized}\r\n\r\n").as_bytes(), + ) + .await?; + // Written in chunks so the bound is hit while the body is still + // arriving, which is the case that matters. + let chunk = vec![b'x'; 64 * 1024]; + let mut sent = 0; + while sent < oversized { + let take = chunk.len().min(oversized - sent); + if socket.write_all(&chunk[..take]).await.is_err() { + break; + } + sent += take; + } + Ok::<_, std::io::Error>(()) + }); + + let err = http_request_bounded( + "GET", + &format!("http://{addr}"), + "/prpc", + b"", + &[], + Some(MAX_RESPONSE_BYTES), + ) + .await + .expect_err("an oversized body must be refused"); + assert!( + format!("{err:#}").contains("exceeds"), + "unexpected error: {err:#}" + ); + Ok(()) + } + + #[tokio::test] + async fn a_response_body_within_the_bound_still_arrives() -> Result<(), Box<dyn Error>> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let size = 128 * 1024; + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + let mut buf = [0u8; 1024]; + let _ = socket.read(&mut buf).await?; + socket + .write_all(format!("HTTP/1.1 200 OK\r\nContent-Length: {size}\r\n\r\n").as_bytes()) + .await?; + socket.write_all(&vec![b'x'; size]).await?; + Ok::<_, std::io::Error>(()) + }); + + let (status, body) = http_request_bounded( + "GET", + &format!("http://{addr}"), + "/prpc", + b"", + &[], + Some(MAX_RESPONSE_BYTES), + ) + .await?; + assert_eq!(status, 200); + assert_eq!(body.len(), size); + Ok(()) + } + + /// The default is unbounded on purpose: `dstack vmm logs --lines 20000` is + /// a legitimate multi-megabyte fetch from a local daemon, and capping it + /// would turn a working command into an error. + #[tokio::test] + async fn an_unbounded_caller_still_gets_a_large_body() -> Result<(), Box<dyn Error>> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let size = MAX_RESPONSE_BYTES + 4096; + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + let mut buf = [0u8; 1024]; + let _ = socket.read(&mut buf).await?; + socket + .write_all(format!("HTTP/1.1 200 OK\r\nContent-Length: {size}\r\n\r\n").as_bytes()) + .await?; + socket.write_all(&vec![b'x'; size]).await?; + Ok::<_, std::io::Error>(()) + }); + + let (status, body) = http_request("GET", &format!("http://{addr}"), "/logs", b"").await?; + assert_eq!(status, 200); + assert_eq!(body.len(), size); + Ok(()) + } + + #[tokio::test] + async fn http_transport_sends_extra_headers() -> Result<(), Box<dyn Error>> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + let mut buf = [0u8; 1024]; + let n = socket.read(&mut buf).await?; + let request = String::from_utf8_lossy(&buf[..n]).into_owned(); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await?; + Ok::<_, std::io::Error>(request) + }); + + let (status, _body) = http_request_with_headers( + "POST", + &format!("http://{addr}"), + "/prpc/Status", + b"{}", + &[("Authorization", "Bearer secret-token")], + ) + .await?; + assert_eq!(status, 200); + let request = server.await??; + assert!( + request.contains("authorization: Bearer secret-token") + || request.contains("Authorization: Bearer secret-token"), + "request is missing the Authorization header: {request:?}" + ); + Ok(()) + } + + #[tokio::test] + async fn http_transport_omits_headers_when_none() -> Result<(), Box<dyn Error>> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + let mut buf = [0u8; 1024]; + let n = socket.read(&mut buf).await?; + let request = String::from_utf8_lossy(&buf[..n]).into_owned(); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await?; + Ok::<_, std::io::Error>(request) + }); + + let (status, _body) = http_request("GET", &format!("http://{addr}"), "/x", b"").await?; + assert_eq!(status, 200); + let request = server.await??; + assert!( + !request.to_lowercase().contains("authorization:"), + "unexpected Authorization header: {request:?}" + ); + Ok(()) + } + + /// A keep-alive server that counts connections and serves requests on each + /// until the peer goes away. Returns its address and the counter. + async fn counting_server() -> (std::net::SocketAddr, std::sync::Arc<AtomicUsize>) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("addr"); + let connections = std::sync::Arc::new(AtomicUsize::new(0)); + let counter = connections.clone(); + tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + counter.fetch_add(1, Ordering::SeqCst); + tokio::spawn(async move { + let mut pending = Vec::new(); + let mut buf = [0u8; 1024]; + loop { + // One response per request, so a pooled client can send + // a second request down the same connection. + while !pending.windows(4).any(|w| w == b"\r\n\r\n") { + match socket.read(&mut buf).await { + Ok(0) | Err(_) => return, + Ok(n) => pending.extend_from_slice(&buf[..n]), + } + } + pending.clear(); + if socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await + .is_err() + { + return; + } + } + }); + } + }); + (addr, connections) + } + + async fn get_thrice(addr: std::net::SocketAddr, connection_reuse: ConnectionReuse) { + for _ in 0..3 { + let (status, _) = http_request_with_options( + "GET", + &format!("http://{addr}"), + "/x", + b"", + &[], + RequestOptions { + connection_reuse, + ..Default::default() + }, + ) + .await + .expect("request should succeed"); + assert_eq!(status, 200); + } + } + + /// The half of the shared-client change that is a behaviour, not a saving. + /// + /// `Fresh` exists so a health probe keeps testing what it used to test back + /// when every request built its own client: that a connection can be opened + /// *now*. Nothing in the type system enforces that + /// `pool_max_idle_per_host(0)` means what the doc comment claims, so this + /// counts connections at the socket. + #[tokio::test] + async fn fresh_dials_every_request_and_pooled_does_not() { + let (addr, connections) = counting_server().await; + get_thrice(addr, ConnectionReuse::Fresh).await; + assert_eq!( + connections.load(Ordering::SeqCst), + 3, + "a probe must open its own connection, or it cannot see an agent \ + that has stopped accepting them" + ); + + let (addr, connections) = counting_server().await; + get_thrice(addr, ConnectionReuse::Pooled).await; + assert_eq!( + connections.load(Ordering::SeqCst), + 1, + "the default should be reusing the connection it already has" + ); + } } diff --git a/dstack/http-client/src/prpc.rs b/dstack/http-client/src/prpc.rs index eacc03b1e..5971a1224 100644 --- a/dstack/http-client/src/prpc.rs +++ b/dstack/http-client/src/prpc.rs @@ -8,10 +8,15 @@ use prpc::{ serde_json, Message, }; use serde::{de::DeserializeOwned, Serialize}; +use std::time::Duration; pub struct PrpcClient { base_url: String, path_append: String, + auth_token: Option<String>, + max_response_bytes: Option<usize>, + request_timeout: Option<Duration>, + connection_reuse: super::ConnectionReuse, } impl PrpcClient { @@ -19,6 +24,10 @@ impl PrpcClient { Self { base_url, path_append: String::new(), + auth_token: None, + max_response_bytes: None, + request_timeout: None, + connection_reuse: Default::default(), } } @@ -29,8 +38,76 @@ impl PrpcClient { Self { base_url: format!("unix:{socket_path}"), path_append: path, + auth_token: None, + max_response_bytes: None, + request_timeout: None, + connection_reuse: Default::default(), } } + + /// Refuse a response larger than `max_bytes`. + /// + /// For callers whose peer is not trusted -- anything talking to a CVM's + /// guest agent. Unbounded otherwise, because most peers here are local or + /// operator-run and return as much as the caller asked for. + pub fn with_max_response_bytes(mut self, max_bytes: usize) -> Self { + self.max_response_bytes = Some(max_bytes); + self + } + + /// The bound in force, if any. Lets a caller assert it set one. + pub fn max_response_bytes(&self) -> Option<usize> { + self.max_response_bytes + } + + /// Choose whether requests may reuse a connection. See + /// [`super::ConnectionReuse`] for when the default is the wrong answer. + pub fn with_connection_reuse(mut self, reuse: super::ConnectionReuse) -> Self { + self.connection_reuse = reuse; + self + } + + /// The policy in force. Lets a caller assert it set the one it meant. + pub fn connection_reuse(&self) -> super::ConnectionReuse { + self.connection_reuse + } + + /// Send `Authorization: Bearer <token>` with every request. + pub fn with_bearer_token(mut self, token: impl Into<String>) -> Self { + let token = token.into(); + self.auth_token = (!token.is_empty()).then_some(token); + self + } + + /// Bound the complete request, including connection establishment, request + /// upload, response headers, and response body. + pub fn with_request_timeout(mut self, timeout: Duration) -> Self { + self.request_timeout = Some(timeout); + self + } +} + +/// The reason a pRPC server gave for failing a JSON request, if it gave one. +/// +/// Servers built on `ra-rpc` answer a failed call with `{"error": "..."}`. +/// Without it the caller learns only a status code, which says that the call +/// failed but not why. +fn server_error_message(body: &[u8]) -> Option<String> { + #[derive(serde::Deserialize)] + struct ErrorBody { + error: String, + } + serde_json::from_slice::<ErrorBody>(body) + .ok() + .map(|body| body.error) +} + +fn normalize_json_response_body(body: &[u8]) -> &[u8] { + if body.is_empty() { + b"null" + } else { + body + } } impl RequestClient for PrpcClient { @@ -41,11 +118,92 @@ impl RequestClient for PrpcClient { { let body = serde_json::to_vec(&body).context("Failed to serialize body")?; let path = format!("{}{path}?json", self.path_append); - let (status, body) = super::http_request("POST", &self.base_url, &path, &body).await?; + let auth_header; + let mut headers: Vec<(&str, &str)> = Vec::new(); + if let Some(token) = &self.auth_token { + auth_header = format!("Bearer {token}"); + headers.push(("Authorization", auth_header.as_str())); + } + let request = super::http_request_with_options( + "POST", + &self.base_url, + &path, + &body, + &headers, + super::RequestOptions { + max_response_bytes: self.max_response_bytes, + connection_reuse: self.connection_reuse, + }, + ); + let (status, body) = match self.request_timeout { + Some(timeout) => tokio::time::timeout(timeout, request) + .await + .context("pRPC request timed out")??, + None => request.await?, + }; if status != 200 { - anyhow::bail!("Invalid status code: {status}, path={path}"); + match server_error_message(&body) { + Some(error) => anyhow::bail!("{path} failed with status {status}: {error}"), + None => anyhow::bail!("Invalid status code: {status}, path={path}"), + } } - let response = serde_json::from_slice(&body).context("Failed to deserialize response")?; + let response = serde_json::from_slice(normalize_json_response_body(&body)) + .context("Failed to deserialize response")?; Ok(response) } } + +#[cfg(test)] +mod response_tests { + use super::{normalize_json_response_body, serde_json, PrpcClient}; + use prpc::client::RequestClient; + use std::{future::pending, time::Duration}; + use tokio::net::TcpListener; + + #[derive(Clone, PartialEq, prpc::Message, serde::Serialize, serde::Deserialize)] + struct Empty {} + + #[test] + fn empty_json_response_decodes_as_unit() { + let value: () = serde_json::from_slice(normalize_json_response_body(b"")) + .expect("empty response should decode as unit"); + assert_eq!(value, ()); + } + + #[test] + fn a_server_error_body_yields_its_message() { + assert_eq!( + super::server_error_message(br#"{"error": "no such bridge"}"#).as_deref(), + Some("no such bridge") + ); + assert_eq!(super::server_error_message(b"<html>502</html>"), None); + } + + #[test] + fn non_empty_json_response_is_unchanged() { + assert_eq!( + normalize_json_response_body(br#"{"value":1}"#), + br#"{"value":1}"# + ); + } + + #[tokio::test] + async fn request_timeout_covers_waiting_for_the_response() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (_stream, _) = listener.accept().await.unwrap(); + pending::<()>().await; + }); + let client = PrpcClient::new(format!("http://{addr}")) + .with_request_timeout(Duration::from_millis(50)); + + let error = client + .request::<Empty, Empty>("Test.Hang", Empty {}) + .await + .unwrap_err(); + + assert!(error.to_string().contains("pRPC request timed out")); + server.abort(); + } +} diff --git a/dstack/kms/Cargo.toml b/dstack/kms/Cargo.toml index c63e407d2..a35f34e7e 100644 --- a/dstack/kms/Cargo.toml +++ b/dstack/kms/Cargo.toml @@ -11,10 +11,9 @@ license.workspace = true [dependencies] anyhow.workspace = true -chrono.workspace = true clap.workspace = true fs-err.workspace = true -git-version.workspace = true +dstack-build-info.workspace = true hex.workspace = true hex_fmt.workspace = true rocket.workspace = true @@ -47,10 +46,11 @@ tempfile.workspace = true serde-duration.workspace = true dstack-verifier = { workspace = true, default-features = false } dstack-mr.workspace = true +dstack-attest.workspace = true +dstack-api-auth.workspace = true [dev-dependencies] cc-eventlog.workspace = true -dstack-attest.workspace = true [features] default = [] diff --git a/dstack/kms/README.md b/dstack/kms/README.md index 35d9a4bda..4ea96fbda 100644 --- a/dstack/kms/README.md +++ b/dstack/kms/README.md @@ -30,6 +30,41 @@ CVMs running in dstack support three boot modes: - `key-provider` in RTMR: `{"type": "kms", "id": "<kms-root-pubkey>"}` - `app-id` is equal to the address of the deployed App Smart Contract. +## Admin API authentication + +The KMS public RPCs authenticate callers by RA-TLS attestation. Operator-facing +admin RPCs (currently `ClearImageCache`) are instead served on a **separate admin +listener** (`[core.admin]` in `kms.toml`) behind the same shared HTTP +authenticator used by the VMM and gateway, configured identically to the gateway +admin API, so the credential travels in the `Authorization: Bearer <token>` or +`X-Admin-Token: <token>` header: + +```toml +[core.admin] +enabled = true +address = "127.0.0.1" +port = 8001 +# generate with: openssl rand -hex 32 +auth_token = "<token>" +# htpasswd_file = "/etc/kms/admin.htpasswd" # bcrypt (htpasswd -B), optional +insecure_no_auth = false +``` + +The token can also be supplied via the `DSTACK_KMS_ADMIN_TOKEN` or +`ADMIN_API_TOKEN` environment variables instead of the config file. The admin +listener fails closed: enabled with neither `auth_token` nor `htpasswd_file` +(and `insecure_no_auth = false`) refuses to start. Call it with, for example: + +```bash +curl -X POST "http://127.0.0.1:8001/prpc/Admin.ClearImageCache?json" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"image_hash": "<hash>", "config_hash": "<hash>"}' +``` + +This admin authentication is separate from the on-chain / webhook authorization +below, which decides whether a CVM may boot and receive keys. + ## KMS Implementation ### Components diff --git a/dstack/kms/auth-eth-bun/bun.lock b/dstack/kms/auth-eth-bun/bun.lock index e94bbb4d5..0af400847 100644 --- a/dstack/kms/auth-eth-bun/bun.lock +++ b/dstack/kms/auth-eth-bun/bun.lock @@ -5,7 +5,7 @@ "name": "auth-eth-bun", "dependencies": { "@hono/zod-validator": "0.2.2", - "hono": "4.12.12", + "hono": "4.12.34", "viem": "2.31.7", "zod": "3.25.76", }, @@ -16,7 +16,7 @@ "openapi-types": "12.1.3", "oxlint": "0.9.10", "typescript": "5.8.3", - "vitest": "1.6.1", + "vitest": "3.2.6", }, }, }, @@ -155,19 +155,27 @@ "@types/bun": ["@types/bun@1.2.18", "", { "dependencies": { "bun-types": "1.2.18" } }, "sha512-Xf6RaWVheyemaThV0kUfaAUvCNokFr+bH8Jxp+tTZfx7dAPA8z9ePnP9S9+Vspzuxxx9JRAXhnyccRj3GyCMdQ=="], + "@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="], + + "@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="], + "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], "@types/node": ["@types/node@24.0.14", "", { "dependencies": { "undici-types": "~7.8.0" } }, "sha512-4zXMWD91vBLGRtHK3YbIoFMia+1nqEz72coM42C5ETjnNCa/heoj7NT1G67iAfOqMmcfhuCZ4uNpyz8EjlAejw=="], "@types/react": ["@types/react@19.1.8", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-AwAfQ2Wa5bCx9WP8nZL2uMZWod7J7/JSplxbTmBQ5ms6QpqNYm672H0Vu9ZVKVngQ+ii4R/byguVEUZQyeg44g=="], - "@vitest/expect": ["@vitest/expect@1.6.1", "", { "dependencies": { "@vitest/spy": "1.6.1", "@vitest/utils": "1.6.1", "chai": "^4.3.10" } }, "sha512-jXL+9+ZNIJKruofqXuuTClf44eSpcHlgj3CiuNihUF3Ioujtmc0zIa3UJOW5RjDK1YLBJZnWBlPuqhYycLioog=="], + "@vitest/expect": ["@vitest/expect@3.2.6", "", { "dependencies": { "@types/chai": "^5.2.2", "@vitest/spy": "3.2.6", "@vitest/utils": "3.2.6", "chai": "^5.2.0", "tinyrainbow": "^2.0.0" } }, "sha512-1+7q9BtaKzEmO+fmNT3kYvoNn5Y71XWAx2Q5HRim4tTVRQVRv4uJFAQ5FbK0OPUeNP/WmVCpxYxoJdvuHVjzBQ=="], + + "@vitest/mocker": ["@vitest/mocker@3.2.6", "", { "dependencies": { "@vitest/spy": "3.2.6", "estree-walker": "^3.0.3", "magic-string": "^0.30.17" }, "peerDependencies": { "msw": "^2.4.9", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" }, "optionalPeers": ["msw", "vite"] }, "sha512-EZOrpDbkKotFAP7wPAQV1UIyoGOk4oX7ynWhBhLB7v+meMHbQhU16oPpIYGTTe4oFlhpryGpgpcZP/sin3hYuw=="], - "@vitest/runner": ["@vitest/runner@1.6.1", "", { "dependencies": { "@vitest/utils": "1.6.1", "p-limit": "^5.0.0", "pathe": "^1.1.1" } }, "sha512-3nSnYXkVkf3mXFfE7vVyPmi3Sazhb/2cfZGGs0JRzFsPFvAMBEcrweV1V1GsrstdXeKCTXlJbvnQwGWgEIHmOA=="], + "@vitest/pretty-format": ["@vitest/pretty-format@3.2.7", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA=="], - "@vitest/snapshot": ["@vitest/snapshot@1.6.1", "", { "dependencies": { "magic-string": "^0.30.5", "pathe": "^1.1.1", "pretty-format": "^29.7.0" } }, "sha512-WvidQuWAzU2p95u8GAKlRMqMyN1yOJkGHnx3M1PL9Raf7AQ1kwLKg04ADlCa3+OXUZE7BceOhVZiuWAbzCKcUQ=="], + "@vitest/runner": ["@vitest/runner@3.2.6", "", { "dependencies": { "@vitest/utils": "3.2.6", "pathe": "^2.0.3", "strip-literal": "^3.0.0" } }, "sha512-HYcoSj1w5tcgUnzoF0HcyaAQjpA1gj9ftUJ7iSJSuipc02jW9gKkigwZbjFldAfYHA1fa8UZVRftdMY5msWM9Q=="], - "@vitest/spy": ["@vitest/spy@1.6.1", "", { "dependencies": { "tinyspy": "^2.2.0" } }, "sha512-MGcMmpGkZebsMZhbQKkAf9CX5zGvjkBTqf8Zx3ApYWXr3wG+QvEu2eXWfnIIWYSJExIp4V9FCKDEeygzkYrXMw=="], + "@vitest/snapshot": ["@vitest/snapshot@3.2.6", "", { "dependencies": { "@vitest/pretty-format": "3.2.6", "magic-string": "^0.30.17", "pathe": "^2.0.3" } }, "sha512-H+ZjNTWGpObenh0YnlBctAPnJSI20P81PL8BPzWpx54YXLLTm8hEsWawtcYLMrwvpK48hGxLLbCS+1KRXhsKhw=="], + + "@vitest/spy": ["@vitest/spy@3.2.6", "", { "dependencies": { "tinyspy": "^4.0.3" } }, "sha512-oq6BbH68WzcWmwtBrU9nqLeaXTR4XwJF7FSLkKEZo4i6eoXcrxjcwSuTvWBIRUTC6VC72nXYunzqgZA+IKdtxg=="], "@vitest/ui": ["@vitest/ui@1.6.1", "", { "dependencies": { "@vitest/utils": "1.6.1", "fast-glob": "^3.3.2", "fflate": "^0.8.1", "flatted": "^3.2.9", "pathe": "^1.1.1", "picocolors": "^1.0.0", "sirv": "^2.0.4" }, "peerDependencies": { "vitest": "1.6.1" } }, "sha512-xa57bCPGuzEFqGjPs3vVLyqareG8DX0uMkr5U/v5vLv5/ZUrBrPL7gzxzTJedEyZxFMfsozwTIbbYfEQVo3kgg=="], @@ -175,13 +183,9 @@ "abitype": ["abitype@1.0.8", "", { "peerDependencies": { "typescript": ">=5.0.4", "zod": "^3 >=3.22.0" }, "optionalPeers": ["typescript", "zod"] }, "sha512-ZeiI6h3GnW06uYDLx0etQtX/p8E24UaHHBj57RSjK7YBFe7iuVn07EDpOeP451D06sF27VOz9JJPlIKJmXgkEg=="], - "acorn": ["acorn@8.15.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="], - - "acorn-walk": ["acorn-walk@8.3.4", "", { "dependencies": { "acorn": "^8.11.0" } }, "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g=="], - "ansi-styles": ["ansi-styles@5.2.0", "", {}, "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA=="], - "assertion-error": ["assertion-error@1.1.0", "", {}, "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw=="], + "assertion-error": ["assertion-error@2.0.1", "", {}, "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA=="], "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], @@ -189,34 +193,34 @@ "cac": ["cac@6.7.14", "", {}, "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ=="], - "chai": ["chai@4.5.0", "", { "dependencies": { "assertion-error": "^1.1.0", "check-error": "^1.0.3", "deep-eql": "^4.1.3", "get-func-name": "^2.0.2", "loupe": "^2.3.6", "pathval": "^1.1.1", "type-detect": "^4.1.0" } }, "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw=="], + "chai": ["chai@5.3.3", "", { "dependencies": { "assertion-error": "^2.0.1", "check-error": "^2.1.1", "deep-eql": "^5.0.1", "loupe": "^3.1.0", "pathval": "^2.0.0" } }, "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw=="], - "check-error": ["check-error@1.0.3", "", { "dependencies": { "get-func-name": "^2.0.2" } }, "sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg=="], - - "confbox": ["confbox@0.1.8", "", {}, "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w=="], - - "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + "check-error": ["check-error@2.1.3", "", {}, "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA=="], "csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="], "debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="], - "deep-eql": ["deep-eql@4.1.4", "", { "dependencies": { "type-detect": "^4.0.0" } }, "sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg=="], + "deep-eql": ["deep-eql@5.0.2", "", {}, "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q=="], "diff-sequences": ["diff-sequences@29.6.3", "", {}, "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q=="], + "es-module-lexer": ["es-module-lexer@1.7.0", "", {}, "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA=="], + "esbuild": ["esbuild@0.21.5", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.21.5", "@esbuild/android-arm": "0.21.5", "@esbuild/android-arm64": "0.21.5", "@esbuild/android-x64": "0.21.5", "@esbuild/darwin-arm64": "0.21.5", "@esbuild/darwin-x64": "0.21.5", "@esbuild/freebsd-arm64": "0.21.5", "@esbuild/freebsd-x64": "0.21.5", "@esbuild/linux-arm": "0.21.5", "@esbuild/linux-arm64": "0.21.5", "@esbuild/linux-ia32": "0.21.5", "@esbuild/linux-loong64": "0.21.5", "@esbuild/linux-mips64el": "0.21.5", "@esbuild/linux-ppc64": "0.21.5", "@esbuild/linux-riscv64": "0.21.5", "@esbuild/linux-s390x": "0.21.5", "@esbuild/linux-x64": "0.21.5", "@esbuild/netbsd-x64": "0.21.5", "@esbuild/openbsd-x64": "0.21.5", "@esbuild/sunos-x64": "0.21.5", "@esbuild/win32-arm64": "0.21.5", "@esbuild/win32-ia32": "0.21.5", "@esbuild/win32-x64": "0.21.5" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw=="], "estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], "eventemitter3": ["eventemitter3@5.0.1", "", {}, "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA=="], - "execa": ["execa@8.0.1", "", { "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^8.0.1", "human-signals": "^5.0.0", "is-stream": "^3.0.0", "merge-stream": "^2.0.0", "npm-run-path": "^5.1.0", "onetime": "^6.0.0", "signal-exit": "^4.1.0", "strip-final-newline": "^3.0.0" } }, "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg=="], + "expect-type": ["expect-type@1.4.0", "", {}, "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA=="], "fast-glob": ["fast-glob@3.3.3", "", { "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.8" } }, "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg=="], "fastq": ["fastq@1.19.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ=="], + "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], + "fflate": ["fflate@0.8.2", "", {}, "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A=="], "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], @@ -227,13 +231,9 @@ "get-func-name": ["get-func-name@2.0.2", "", {}, "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ=="], - "get-stream": ["get-stream@8.0.1", "", {}, "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA=="], - "glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], - "hono": ["hono@4.12.12", "", {}, "sha512-p1JfQMKaceuCbpJKAPKVqyqviZdS0eUxH9v82oWo1kb9xjQ5wA6iP3FNVAPDFlz5/p7d45lO+BpSk1tuSZMF4Q=="], - - "human-signals": ["human-signals@5.0.0", "", {}, "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ=="], + "hono": ["hono@4.12.34", "", {}, "sha512-GqXJqY/xJkJmuloTrnV1ZEXG3fqte+VjkUqoRNZXcrUidiUOP4fMSIHHY4tsqZBK++kVyWmt/AAfSUuy57/eSA=="], "is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="], @@ -241,59 +241,37 @@ "is-number": ["is-number@7.0.0", "", {}, "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="], - "is-stream": ["is-stream@3.0.0", "", {}, "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA=="], - - "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], - "isows": ["isows@1.0.7", "", { "peerDependencies": { "ws": "*" } }, "sha512-I1fSfDCZL5P0v33sVqeTDSpcstAg/N+wF5HS033mogOVIp4B+oHC7oOCsA3axAbBSGTJ8QubbNmnIRN/h8U7hg=="], "js-tokens": ["js-tokens@9.0.1", "", {}, "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ=="], - "local-pkg": ["local-pkg@0.5.1", "", { "dependencies": { "mlly": "^1.7.3", "pkg-types": "^1.2.1" } }, "sha512-9rrA30MRRP3gBD3HTGnC6cDFpaE1kVDWxWgqWJUN0RvDNAo+Nz/9GxB+nHOH0ifbVFy0hSA1V6vFDvnx54lTEQ=="], - "loupe": ["loupe@2.3.7", "", { "dependencies": { "get-func-name": "^2.0.1" } }, "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA=="], "magic-string": ["magic-string@0.30.17", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" } }, "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA=="], - "merge-stream": ["merge-stream@2.0.0", "", {}, "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="], - "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], - "mimic-fn": ["mimic-fn@4.0.0", "", {}, "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw=="], - - "mlly": ["mlly@1.7.4", "", { "dependencies": { "acorn": "^8.14.0", "pathe": "^2.0.1", "pkg-types": "^1.3.0", "ufo": "^1.5.4" } }, "sha512-qmdSIPC4bDJXgZTCR7XosJiNKySV7O215tsPtDN9iEO/7q/76b/ijtgRu/+epFXSJhijtTCCGp3DWS549P3xKw=="], - "mrmime": ["mrmime@2.0.1", "", {}, "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ=="], "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], - "npm-run-path": ["npm-run-path@5.3.0", "", { "dependencies": { "path-key": "^4.0.0" } }, "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ=="], - - "onetime": ["onetime@6.0.0", "", { "dependencies": { "mimic-fn": "^4.0.0" } }, "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ=="], - "openapi-types": ["openapi-types@12.1.3", "", {}, "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw=="], "ox": ["ox@0.8.1", "", { "dependencies": { "@adraffy/ens-normalize": "^1.11.0", "@noble/ciphers": "^1.3.0", "@noble/curves": "^1.9.1", "@noble/hashes": "^1.8.0", "@scure/bip32": "^1.7.0", "@scure/bip39": "^1.6.0", "abitype": "^1.0.8", "eventemitter3": "5.0.1" }, "peerDependencies": { "typescript": ">=5.4.0" }, "optionalPeers": ["typescript"] }, "sha512-e+z5epnzV+Zuz91YYujecW8cF01mzmrUtWotJ0oEPym/G82uccs7q0WDHTYL3eiONbTUEvcZrptAKLgTBD3u2A=="], "oxlint": ["oxlint@0.9.10", "", { "optionalDependencies": { "@oxlint/darwin-arm64": "0.9.10", "@oxlint/darwin-x64": "0.9.10", "@oxlint/linux-arm64-gnu": "0.9.10", "@oxlint/linux-arm64-musl": "0.9.10", "@oxlint/linux-x64-gnu": "0.9.10", "@oxlint/linux-x64-musl": "0.9.10", "@oxlint/win32-arm64": "0.9.10", "@oxlint/win32-x64": "0.9.10" }, "bin": { "oxlint": "bin/oxlint", "oxc_language_server": "bin/oxc_language_server" } }, "sha512-bKiiFN7Hnoaist/rditTRBXz+GXKYuLd53/NB7Q6zHB/bifELJarSoRLkAUGElIJKl4PSr3lTh1g6zehh+rX0g=="], - "p-limit": ["p-limit@5.0.0", "", { "dependencies": { "yocto-queue": "^1.0.0" } }, "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ=="], - - "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], - "pathe": ["pathe@1.1.2", "", {}, "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ=="], - "pathval": ["pathval@1.1.1", "", {}, "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ=="], + "pathval": ["pathval@2.0.1", "", {}, "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ=="], "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], - "picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], - - "pkg-types": ["pkg-types@1.3.1", "", { "dependencies": { "confbox": "^0.1.8", "mlly": "^1.7.4", "pathe": "^2.0.1" } }, "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ=="], + "picomatch": ["picomatch@4.0.5", "", {}, "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A=="], "postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="], @@ -309,14 +287,8 @@ "run-parallel": ["run-parallel@1.2.0", "", { "dependencies": { "queue-microtask": "^1.2.2" } }, "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA=="], - "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], - - "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], - "siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="], - "signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], - "sirv": ["sirv@2.0.4", "", { "dependencies": { "@polka/url": "^1.0.0-next.24", "mrmime": "^2.0.0", "totalist": "^3.0.0" } }, "sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ=="], "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], @@ -325,54 +297,76 @@ "std-env": ["std-env@3.9.0", "", {}, "sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw=="], - "strip-final-newline": ["strip-final-newline@3.0.0", "", {}, "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw=="], - - "strip-literal": ["strip-literal@2.1.1", "", { "dependencies": { "js-tokens": "^9.0.1" } }, "sha512-631UJ6O00eNGfMiWG78ck80dfBab8X6IVFB51jZK5Icd7XAs60Z5y7QdSd/wGIklnWvRbUNloVzhOKKmutxQ6Q=="], + "strip-literal": ["strip-literal@3.1.0", "", { "dependencies": { "js-tokens": "^9.0.1" } }, "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg=="], "tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="], - "tinypool": ["tinypool@0.8.4", "", {}, "sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ=="], + "tinyexec": ["tinyexec@0.3.2", "", {}, "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA=="], + + "tinyglobby": ["tinyglobby@0.2.17", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="], + + "tinypool": ["tinypool@1.1.1", "", {}, "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg=="], + + "tinyrainbow": ["tinyrainbow@2.0.0", "", {}, "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw=="], - "tinyspy": ["tinyspy@2.2.1", "", {}, "sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A=="], + "tinyspy": ["tinyspy@4.0.4", "", {}, "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q=="], "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], "totalist": ["totalist@3.0.1", "", {}, "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ=="], - "type-detect": ["type-detect@4.1.0", "", {}, "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw=="], - "typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="], - "ufo": ["ufo@1.6.1", "", {}, "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA=="], - "undici-types": ["undici-types@7.8.0", "", {}, "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw=="], "viem": ["viem@2.31.7", "", { "dependencies": { "@noble/curves": "1.9.2", "@noble/hashes": "1.8.0", "@scure/bip32": "1.7.0", "@scure/bip39": "1.6.0", "abitype": "1.0.8", "isows": "1.0.7", "ox": "0.8.1", "ws": "8.18.2" }, "peerDependencies": { "typescript": ">=5.0.4" }, "optionalPeers": ["typescript"] }, "sha512-mpB8Hp6xK77E/b/yJmpAIQcxcOfpbrwWNItjnXaIA8lxZYt4JS433Pge2gg6Hp3PwyFtaUMh01j5L8EXnLTjQQ=="], "vite": ["vite@5.4.19", "", { "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", "rollup": "^4.20.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || >=20.0.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" }, "optionalPeers": ["@types/node", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser"], "bin": { "vite": "bin/vite.js" } }, "sha512-qO3aKv3HoQC8QKiNSTuUM1l9o/XX3+c+VTgLHbJWHZGeTPVAg2XwazI9UWzoxjIJCGCV2zU60uqMzjeLZuULqA=="], - "vite-node": ["vite-node@1.6.1", "", { "dependencies": { "cac": "^6.7.14", "debug": "^4.3.4", "pathe": "^1.1.1", "picocolors": "^1.0.0", "vite": "^5.0.0" }, "bin": { "vite-node": "vite-node.mjs" } }, "sha512-YAXkfvGtuTzwWbDSACdJSg4A4DZiAqckWe90Zapc/sEX3XvHcw1NdurM/6od8J207tSDqNbSsgdCacBgvJKFuA=="], - - "vitest": ["vitest@1.6.1", "", { "dependencies": { "@vitest/expect": "1.6.1", "@vitest/runner": "1.6.1", "@vitest/snapshot": "1.6.1", "@vitest/spy": "1.6.1", "@vitest/utils": "1.6.1", "acorn-walk": "^8.3.2", "chai": "^4.3.10", "debug": "^4.3.4", "execa": "^8.0.1", "local-pkg": "^0.5.0", "magic-string": "^0.30.5", "pathe": "^1.1.1", "picocolors": "^1.0.0", "std-env": "^3.5.0", "strip-literal": "^2.0.0", "tinybench": "^2.5.1", "tinypool": "^0.8.3", "vite": "^5.0.0", "vite-node": "1.6.1", "why-is-node-running": "^2.2.2" }, "peerDependencies": { "@edge-runtime/vm": "*", "@types/node": "^18.0.0 || >=20.0.0", "@vitest/browser": "1.6.1", "@vitest/ui": "1.6.1", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@types/node", "@vitest/browser", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-Ljb1cnSJSivGN0LqXd/zmDbWEM0RNNg2t1QW/XUhYl/qPqyu7CsqeWtqQXHVaJsecLPuDoak2oJcZN2QoRIOag=="], + "vite-node": ["vite-node@3.2.4", "", { "dependencies": { "cac": "^6.7.14", "debug": "^4.4.1", "es-module-lexer": "^1.7.0", "pathe": "^2.0.3", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" }, "bin": { "vite-node": "vite-node.mjs" } }, "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg=="], - "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], + "vitest": ["vitest@3.2.6", "", { "dependencies": { "@types/chai": "^5.2.2", "@vitest/expect": "3.2.6", "@vitest/mocker": "3.2.6", "@vitest/pretty-format": "^3.2.6", "@vitest/runner": "3.2.6", "@vitest/snapshot": "3.2.6", "@vitest/spy": "3.2.6", "@vitest/utils": "3.2.6", "chai": "^5.2.0", "debug": "^4.4.1", "expect-type": "^1.2.1", "magic-string": "^0.30.17", "pathe": "^2.0.3", "picomatch": "^4.0.2", "std-env": "^3.9.0", "tinybench": "^2.9.0", "tinyexec": "^0.3.2", "tinyglobby": "^0.2.14", "tinypool": "^1.1.1", "tinyrainbow": "^2.0.0", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", "vite-node": "3.2.4", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@types/debug": "^4.1.12", "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "@vitest/browser": "3.2.6", "@vitest/ui": "3.2.6", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@types/debug", "@types/node", "@vitest/browser", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-xejya+bT/j/+R/AGa1XOfRxLmNUlLtlwjRsFUILF+xHfzElmGcmFydy2gqqIrd62ptIEfwVMofd19uNWD9L7Nw=="], "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], "ws": ["ws@8.18.2", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ=="], - "yocto-queue": ["yocto-queue@1.2.1", "", {}, "sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg=="], - "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + "@vitest/expect/@vitest/utils": ["@vitest/utils@3.2.6", "", { "dependencies": { "@vitest/pretty-format": "3.2.6", "loupe": "^3.1.4", "tinyrainbow": "^2.0.0" } }, "sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg=="], + + "@vitest/runner/@vitest/utils": ["@vitest/utils@3.2.6", "", { "dependencies": { "@vitest/pretty-format": "3.2.6", "loupe": "^3.1.4", "tinyrainbow": "^2.0.0" } }, "sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg=="], + + "@vitest/runner/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + + "@vitest/snapshot/@vitest/pretty-format": ["@vitest/pretty-format@3.2.6", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA=="], + + "@vitest/snapshot/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + "bun-types/@types/node": ["@types/node@22.7.5", "", { "dependencies": { "undici-types": "~6.19.2" } }, "sha512-jML7s2NAzMWc//QSJ1a3prpk78cOPchGvXJsC3C6R6PSMoooztvRVQEz89gmBTBY1SPMaqo5teB4uNHPdetShQ=="], - "mlly/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + "chai/loupe": ["loupe@3.2.1", "", {}, "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ=="], + + "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + + "vite-node/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + + "vitest/@vitest/utils": ["@vitest/utils@3.2.6", "", { "dependencies": { "@vitest/pretty-format": "3.2.6", "loupe": "^3.1.4", "tinyrainbow": "^2.0.0" } }, "sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg=="], - "npm-run-path/path-key": ["path-key@4.0.0", "", {}, "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ=="], + "vitest/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], - "pkg-types/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + "@vitest/expect/@vitest/utils/@vitest/pretty-format": ["@vitest/pretty-format@3.2.6", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA=="], + + "@vitest/expect/@vitest/utils/loupe": ["loupe@3.2.1", "", {}, "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ=="], + + "@vitest/runner/@vitest/utils/@vitest/pretty-format": ["@vitest/pretty-format@3.2.6", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA=="], + + "@vitest/runner/@vitest/utils/loupe": ["loupe@3.2.1", "", {}, "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ=="], "bun-types/@types/node/undici-types": ["undici-types@6.19.8", "", {}, "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw=="], + + "vitest/@vitest/utils/@vitest/pretty-format": ["@vitest/pretty-format@3.2.6", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA=="], + + "vitest/@vitest/utils/loupe": ["loupe@3.2.1", "", {}, "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ=="], } } diff --git a/dstack/kms/auth-eth-bun/index.test.ts b/dstack/kms/auth-eth-bun/index.test.ts index 07f4c2ad7..fd97ffdb6 100644 --- a/dstack/kms/auth-eth-bun/index.test.ts +++ b/dstack/kms/auth-eth-bun/index.test.ts @@ -8,11 +8,13 @@ import openApiSpec from './openapi.json'; // Mock viem const mockReadContract = vi.fn(); const mockGetChainId = vi.fn(); +const mockGetBlockNumber = vi.fn(); vi.mock('viem', () => ({ createPublicClient: vi.fn(() => ({ readContract: mockReadContract, getChainId: mockGetChainId, + getBlockNumber: mockGetBlockNumber, })), http: vi.fn(), getContract: vi.fn(), @@ -26,6 +28,8 @@ beforeAll(async () => { process.env.ETH_RPC_URL = 'http://localhost:8545'; process.env.KMS_CONTRACT_ADDR = '0x1234567890123456789012345678901234567890'; process.env.PORT = '3001'; + process.env.ETH_CHAIN_ID = '1337'; + process.env.ETH_FINALITY_CONFIRMATIONS = '2'; // Import the app after mocking const indexModule = await import('./index.ts'); @@ -35,6 +39,8 @@ beforeAll(async () => { beforeEach(() => { // Reset mocks before each test vi.clearAllMocks(); + mockGetChainId.mockResolvedValue(1337); + mockGetBlockNumber.mockResolvedValue(100n); }); describe('API Compatibility Tests', () => { @@ -85,7 +91,7 @@ describe('API Compatibility Tests', () => { expect(response.status).toBe(500); expect(data).toMatchObject({ status: 'error', - message: expect.any(String), + message: 'authorization backend unavailable', }); }); }); @@ -180,7 +186,7 @@ describe('API Compatibility Tests', () => { expect(data).toMatchObject({ isAllowed: false, gatewayAppId: '', - reason: 'contract call failed', + reason: 'authorization backend unavailable', }); }); @@ -197,7 +203,22 @@ describe('API Compatibility Tests', () => { expect(response.status).toBe(400); }); - }); + + + it('should reject oversized and non-hex measurements before backend use', async () => { + for (const mrAggregated of ['0x' + 'ab'.repeat(33), 'not-hex']) { + const response = await appFetch(new Request('http://localhost:3001/bootAuth/app', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ...validBootInfo, mrAggregated }), + })); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ isAllowed: false, reason: 'invalid authorization request', gatewayAppId: '' }); + } + expect(mockReadContract).not.toHaveBeenCalled(); + }); +}); describe('POST /bootAuth/kms', () => { const validBootInfo = { @@ -277,7 +298,7 @@ describe('API Compatibility Tests', () => { expect(response.status).toBe(200); expect(data.isAllowed).toBe(false); - expect(data.reason).toBe('Test backend error'); + expect(data.reason).toBe('authorization backend unavailable'); // Verify that console.error was not called for test errors expect(consoleSpy).not.toHaveBeenCalled(); @@ -300,10 +321,11 @@ describe('API Compatibility Tests', () => { expect(response.status).toBe(200); expect(data.isAllowed).toBe(false); - expect(data.reason).toBe('real error'); + expect(data.reason).toBe('authorization backend unavailable'); - // Verify that console.error was called for real errors - expect(consoleSpy).toHaveBeenCalledWith('error in KMS boot auth:', expect.any(Error)); + // Diagnostics identify the failing boundary without retaining backend details. + expect(consoleSpy).toHaveBeenCalledWith('KMS authorization backend failed'); + expect(JSON.stringify(consoleSpy.mock.calls)).not.toContain('real error'); consoleSpy.mockRestore(); }); @@ -388,3 +410,158 @@ describe('Hex Decoding Compatibility', () => { expect(response.status).toBe(200); }); }); + +describe('Authorization freshness and domain binding', () => { + const requestBody = { + mrAggregated: '0x' + '11'.repeat(32), + osImageHash: '0x' + '22'.repeat(32), + appId: '0x' + '33'.repeat(20), + composeHash: '0x' + '44'.repeat(32), + instanceId: '0x' + '55'.repeat(20), + deviceId: '0x' + '66'.repeat(32), + }; + + const postApp = (body = requestBody) => appFetch(new Request( + 'http://localhost:3001/bootAuth/app', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }, + )); + + it('re-evaluates replayed payloads instead of caching an earlier allow', async () => { + let decisions = 0; + mockReadContract.mockImplementation((params) => { + expect(params.address).toBe('0x1234567890123456789012345678901234567890'); + if (params.functionName === 'isAppAllowed') { + decisions += 1; + return decisions === 1 ? [true, 'initial allow'] : [false, 'policy changed']; + } + if (params.functionName === 'gatewayAppId') return 'gateway-app'; + throw new Error(`unexpected function ${params.functionName}`); + }); + + const first = await postApp(); + const replay = await postApp(); + + expect(await first.json()).toMatchObject({ isAllowed: true, reason: 'initial allow' }); + expect(await replay.json()).toMatchObject({ isAllowed: false, reason: 'policy changed' }); + expect(decisions).toBe(2); + }); + + it('binds changed measurements and identities into distinct contract arguments', async () => { + const calls: unknown[] = []; + mockReadContract.mockImplementation((params) => { + if (params.functionName === 'isAppAllowed') { + calls.push(params.args[0]); + return [true, 'allowed']; + } + if (params.functionName === 'gatewayAppId') return 'gateway-app'; + throw new Error(`unexpected function ${params.functionName}`); + }); + + await postApp(); + await postApp({ ...requestBody, composeHash: '0x' + '77'.repeat(32) }); + await postApp({ ...requestBody, appId: '0x' + '88'.repeat(20) }); + + expect(calls).toHaveLength(3); + expect(calls[0]).not.toEqual(calls[1]); + expect(calls[0]).not.toEqual(calls[2]); + }); + + it('fails closed during backend interruption and succeeds after recovery', async () => { + mockReadContract.mockRejectedValueOnce(new Error('backend unavailable')); + const interrupted = await postApp(); + expect(await interrupted.json()).toEqual({ + isAllowed: false, + gatewayAppId: '', + reason: 'authorization backend unavailable', + }); + + mockReadContract.mockImplementation((params) => { + if (params.functionName === 'isAppAllowed') return [true, 'recovered']; + if (params.functionName === 'gatewayAppId') return 'gateway-app'; + throw new Error(`unexpected function ${params.functionName}`); + }); + const recovered = await postApp(); + expect(await recovered.json()).toMatchObject({ isAllowed: true, reason: 'recovered' }); + }); +}); + + +describe('Ethereum finalized snapshot authorization', () => { + const requestBody = { + mrAggregated: '0x' + '11'.repeat(32), + osImageHash: '0x' + '22'.repeat(32), + appId: '0x' + '33'.repeat(20), + composeHash: '0x' + '44'.repeat(32), + instanceId: '0x' + '55'.repeat(20), + deviceId: '0x' + '66'.repeat(32), + }; + + const authorize = () => appFetch(new Request('http://localhost:3001/bootAuth/app', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(requestBody), + })); + + it('reads the decision and gateway identity from one confirmation-depth snapshot', async () => { + mockGetBlockNumber.mockResolvedValue(100n); + mockReadContract.mockImplementation((params) => { + expect(params.blockNumber).toBe(98n); + if (params.functionName === 'isAppAllowed') return [true, 'finalized allow']; + if (params.functionName === 'gatewayAppId') return 'gateway-app'; + throw new Error(`unexpected function ${params.functionName}`); + }); + + const response = await authorize(); + expect(await response.json()).toMatchObject({ isAllowed: true, reason: 'finalized allow' }); + expect(mockGetBlockNumber).toHaveBeenCalledTimes(1); + }); + + it('re-evaluates the canonical finalized snapshot after a short reorg', async () => { + mockGetBlockNumber.mockResolvedValueOnce(100n).mockResolvedValueOnce(101n); + let decisions = 0; + const observedBlocks: bigint[] = []; + mockReadContract.mockImplementation((params) => { + if (params.functionName === 'isAppAllowed') { + observedBlocks.push(params.blockNumber); + decisions += 1; + return decisions === 1 ? [true, 'old canonical allow'] : [false, 'new canonical deny']; + } + if (params.functionName === 'gatewayAppId') return 'gateway-app'; + throw new Error(`unexpected function ${params.functionName}`); + }); + + const before = await authorize(); + const after = await authorize(); + expect(await before.json()).toMatchObject({ isAllowed: true }); + expect(await after.json()).toMatchObject({ isAllowed: false, reason: 'new canonical deny' }); + expect(observedBlocks).toEqual([98n, 99n]); + }); + + it.each([ + ['wrong chain', () => mockGetChainId.mockResolvedValue(1)], + ['stale head', () => mockGetBlockNumber.mockResolvedValue(1n)], + ['head timeout', () => mockGetBlockNumber.mockRejectedValue(new Error('timeout'))], + ])('fails closed for %s and recovers without retained decisions', async (_name, inject) => { + inject(); + const failed = await authorize(); + expect(await failed.json()).toEqual({ + isAllowed: false, + gatewayAppId: '', + reason: 'authorization backend unavailable', + }); + + mockGetChainId.mockResolvedValue(1337); + mockGetBlockNumber.mockResolvedValue(102n); + mockReadContract.mockImplementation((params) => { + if (params.functionName === 'isAppAllowed') return [true, 'recovered']; + if (params.functionName === 'gatewayAppId') return 'gateway-app'; + throw new Error(`unexpected function ${params.functionName}`); + }); + const recovered = await authorize(); + expect(await recovered.json()).toMatchObject({ isAllowed: true, reason: 'recovered' }); + }); +}); diff --git a/dstack/kms/auth-eth-bun/index.ts b/dstack/kms/auth-eth-bun/index.ts index b2e0c4158..3ea279327 100644 --- a/dstack/kms/auth-eth-bun/index.ts +++ b/dstack/kms/auth-eth-bun/index.ts @@ -8,18 +8,26 @@ import { z } from 'zod'; import { createPublicClient, http, type Address, type Hex } from 'viem'; // zod schemas for validation - compatible with original fastify implementation +const boundedHex = (bytes: number, description: string) => + z.string() + .regex(/^(?:0x)?[0-9a-fA-F]*$/, `${description} must be hexadecimal`) + .refine( + (value) => value.replace(/^0x/, '').length <= bytes * 2, + `${description} exceeds ${bytes} bytes`, + ); + const BootInfoSchema = z.object({ - // required fields (matching original fastify schema) - mrAggregated: z.string().describe('aggregated MR measurement'), - osImageHash: z.string().describe('OS Image hash'), - appId: z.string().describe('application ID'), - composeHash: z.string().describe('compose hash'), - instanceId: z.string().describe('instance ID'), - deviceId: z.string().describe('device ID'), - // optional fields (for full compatibility with BootInfo interface) - tcbStatus: z.string().optional().default(''), - advisoryIds: z.array(z.string()).optional().default([]), - mrSystem: z.string().optional().default('') + // Short hexadecimal values remain compatible with the original backend, + // which left-pads them before making the contract call. + mrAggregated: boundedHex(32, 'aggregated MR measurement'), + osImageHash: boundedHex(32, 'OS Image hash'), + appId: boundedHex(20, 'application ID'), + composeHash: boundedHex(32, 'compose hash'), + instanceId: boundedHex(20, 'instance ID'), + deviceId: boundedHex(32, 'device ID'), + tcbStatus: z.string().max(128).optional().default(''), + advisoryIds: z.array(z.string().max(256)).max(128).optional().default([]), + mrSystem: boundedHex(32, 'system MR measurement').optional().default('') }); const BootResponseSchema = z.object({ @@ -105,10 +113,31 @@ const DSTACK_KMS_ABI = [ class EthereumBackend { private client: ReturnType<typeof createPublicClient>; private kmsContractAddr: Address; + private expectedChainId?: number; + private finalityConfirmations: bigint; - constructor(client: ReturnType<typeof createPublicClient>, kmsContractAddr: string) { + constructor( + client: ReturnType<typeof createPublicClient>, + kmsContractAddr: string, + expectedChainId: number | undefined, + finalityConfirmations: bigint, + ) { this.client = client; this.kmsContractAddr = kmsContractAddr as Address; + this.expectedChainId = expectedChainId; + this.finalityConfirmations = finalityConfirmations; + } + + private async finalizedBlockNumber(): Promise<bigint> { + const chainId = await this.client.getChainId(); + if (this.expectedChainId !== undefined && chainId !== this.expectedChainId) { + throw new Error('authorization backend chain ID mismatch'); + } + const head = await this.client.getBlockNumber(); + if (head < this.finalityConfirmations) { + throw new Error('authorization backend has not reached configured finality'); + } + return head - this.finalityConfirmations; } private decodeHex(hex: string, sz: number = 32): Hex { @@ -134,20 +163,23 @@ class EthereumBackend { advisoryIds: bootInfo.advisoryIds || [] }; + const blockNumber = await this.finalizedBlockNumber(); let response; if (isKms) { response = await this.client.readContract({ address: this.kmsContractAddr, abi: DSTACK_KMS_ABI, functionName: 'isKmsAllowed', - args: [bootInfoStruct] + args: [bootInfoStruct], + blockNumber }); } else { response = await this.client.readContract({ address: this.kmsContractAddr, abi: DSTACK_KMS_ABI, functionName: 'isAppAllowed', - args: [bootInfoStruct] + args: [bootInfoStruct], + blockNumber }); } @@ -155,7 +187,8 @@ class EthereumBackend { const gatewayAppId = await this.client.readContract({ address: this.kmsContractAddr, abi: DSTACK_KMS_ABI, - functionName: 'gatewayAppId' + functionName: 'gatewayAppId', + blockNumber }); return { @@ -195,10 +228,33 @@ const app = new Hono(); // initialize ethereum backend const rpcUrl = process.env.ETH_RPC_URL || 'http://localhost:8545'; const kmsContractAddr = process.env.KMS_CONTRACT_ADDR || '0x0000000000000000000000000000000000000000'; +const parseNonNegativeInteger = (name: string, value: string | undefined): number | undefined => { + if (value === undefined || value === '') return undefined; + if (!/^(?:0|[1-9][0-9]*)$/.test(value)) throw new Error(`${name} must be a non-negative integer`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) throw new Error(`${name} exceeds the safe integer range`); + return parsed; +}; +const expectedChainId = parseNonNegativeInteger('ETH_CHAIN_ID', process.env.ETH_CHAIN_ID); +const finalityConfirmations = BigInt( + parseNonNegativeInteger('ETH_FINALITY_CONFIRMATIONS', process.env.ETH_FINALITY_CONFIRMATIONS) ?? 0, +); const client = createPublicClient({ transport: http(rpcUrl) }); -const ethereum = new EthereumBackend(client, kmsContractAddr); +const ethereum = new EthereumBackend(client, kmsContractAddr, expectedChainId, finalityConfirmations); + +const publicRpcEndpoint = (value: string): string => { + try { + const endpoint = new URL(value); + return `${endpoint.protocol}//${endpoint.host}`; + } catch { + return 'configured'; + } +}; + +const backendUnavailable = 'authorization backend unavailable'; +const invalidRequest = { isAllowed: false, reason: 'invalid authorization request', gatewayAppId: '' }; // health check and info endpoint app.get('/', async (c) => { @@ -213,35 +269,37 @@ app.get('/', async (c) => { return c.json({ status: 'ok', kmsContractAddr: kmsContractAddr, - ethRpcUrl: rpcUrl, + ethRpcUrl: publicRpcEndpoint(rpcUrl), gatewayAppId: batch[0], chainId: batch[1], appAuthImplementation: batch[2], // NOTE: for backward compatibility appImplementation: batch[2], }); } catch (error) { - console.error('error in health check:', error); + console.error('authorization backend health check failed'); return c.json({ status: 'error', - message: error instanceof Error ? error.message : String(error) + message: backendUnavailable }, 500); } }); // app boot authentication app.post('/bootAuth/app', - zValidator('json', BootInfoSchema), + zValidator('json', BootInfoSchema, (result, c) => { + if (!result.success) return c.json(invalidRequest, 400); + }), async (c) => { try { const bootInfo = c.req.valid('json'); const result = await ethereum.checkBoot(bootInfo, false); return c.json(result); } catch (error) { - console.error('error in app boot auth:', error); + console.error('application authorization backend failed'); return c.json({ isAllowed: false, gatewayAppId: '', - reason: error instanceof Error ? error.message : String(error) + reason: backendUnavailable }); } } @@ -249,7 +307,9 @@ app.post('/bootAuth/app', // KMS boot authentication app.post('/bootAuth/kms', - zValidator('json', BootInfoSchema), + zValidator('json', BootInfoSchema, (result, c) => { + if (!result.success) return c.json(invalidRequest, 400); + }), async (c) => { try { const bootInfo = c.req.valid('json'); @@ -258,12 +318,12 @@ app.post('/bootAuth/kms', } catch (error) { // don't log test backend errors if (!(error instanceof Error && "Test backend error" === error.message)) { - console.error('error in KMS boot auth:', error); + console.error('KMS authorization backend failed'); } return c.json({ isAllowed: false, gatewayAppId: '', - reason: error instanceof Error ? error.message : String(error) + reason: backendUnavailable }); } } diff --git a/dstack/kms/auth-eth-bun/package.json b/dstack/kms/auth-eth-bun/package.json index 62bfba044..1957b4a1d 100644 --- a/dstack/kms/auth-eth-bun/package.json +++ b/dstack/kms/auth-eth-bun/package.json @@ -15,7 +15,7 @@ "check": "bun run lint && bun run test:run" }, "dependencies": { - "hono": "4.12.25", + "hono": "4.13.5", "@hono/zod-validator": "0.2.2", "zod": "3.25.76", "viem": "2.31.7" @@ -27,7 +27,7 @@ "openapi-types": "12.1.3", "oxlint": "0.9.10", "typescript": "5.8.3", - "vitest": "3.2.6" + "vitest": "4.1.11" }, "type": "module" } diff --git a/dstack/kms/auth-eth/contracts/DstackApp.sol b/dstack/kms/auth-eth/contracts/DstackApp.sol index 762731f60..6c27e9ccb 100644 --- a/dstack/kms/auth-eth/contracts/DstackApp.sol +++ b/dstack/kms/auth-eth/contracts/DstackApp.sol @@ -40,6 +40,8 @@ contract DstackApp is event UpgradesDisabled(); event AllowAnyDeviceSet(bool allowAny); event RequireTcbUpToDateSet(bool requireUpToDate); + /// @notice Additive audit event for reconstructing authorization policy. + event PolicyChanged(address indexed actor, bytes32 indexed policy, bytes32 indexed value, bool enabled); /// @custom:oz-upgrades-unsafe-allow constructor constructor() { @@ -94,12 +96,14 @@ contract DstackApp is if (initialDeviceId != bytes32(0)) { allowedDeviceIds[initialDeviceId] = true; emit DeviceAdded(initialDeviceId); + _emitPolicy("device", initialDeviceId, true); } // Add initial compose hash if provided if (initialComposeHash != bytes32(0)) { allowedComposeHashes[initialComposeHash] = true; emit ComposeHashAdded(initialComposeHash); + _emitPolicy("compose-hash", initialComposeHash, true); } __Ownable_init(initialOwner); @@ -130,44 +134,55 @@ contract DstackApp is } // Function to authorize upgrades (required by UUPSUpgradeable) - function _authorizeUpgrade(address) internal view override onlyOwner { + function _authorizeUpgrade(address newImplementation) internal override onlyOwner { require(!_upgradesDisabled, "Upgrades are permanently disabled"); + _emitPolicy("implementation-upgrade", bytes32(uint256(uint160(newImplementation))), true); + } + + function _emitPolicy(string memory policy, bytes32 value, bool enabled) internal { + emit PolicyChanged(msg.sender, keccak256(bytes(policy)), value, enabled); } // Add a compose hash to allowed list function addComposeHash(bytes32 composeHash) external onlyOwner { allowedComposeHashes[composeHash] = true; emit ComposeHashAdded(composeHash); + _emitPolicy("compose-hash", composeHash, true); } // Remove a compose hash from allowed list function removeComposeHash(bytes32 composeHash) external onlyOwner { allowedComposeHashes[composeHash] = false; emit ComposeHashRemoved(composeHash); + _emitPolicy("compose-hash", composeHash, false); } // Set whether any device is allowed to boot this app function setAllowAnyDevice(bool _allowAnyDevice) external onlyOwner { allowAnyDevice = _allowAnyDevice; emit AllowAnyDeviceSet(_allowAnyDevice); + _emitPolicy("allow-any-device", bytes32(0), _allowAnyDevice); } // Set whether TCB status must be UpToDate to boot this app function setRequireTcbUpToDate(bool _requireUpToDate) external onlyOwner { requireTcbUpToDate = _requireUpToDate; emit RequireTcbUpToDateSet(_requireUpToDate); + _emitPolicy("require-tcb-up-to-date", bytes32(0), _requireUpToDate); } // Add a device ID to allowed list function addDevice(bytes32 deviceId) external onlyOwner { allowedDeviceIds[deviceId] = true; emit DeviceAdded(deviceId); + _emitPolicy("device", deviceId, true); } // Remove a device ID from allowed list function removeDevice(bytes32 deviceId) external onlyOwner { allowedDeviceIds[deviceId] = false; emit DeviceRemoved(deviceId); + _emitPolicy("device", deviceId, false); } // Check if an app is allowed to boot @@ -202,6 +217,7 @@ contract DstackApp is function disableUpgrades() external onlyOwner { _upgradesDisabled = true; emit UpgradesDisabled(); + _emitPolicy("upgrades-disabled", bytes32(0), true); } // Add storage gap for upgradeable contracts diff --git a/dstack/kms/auth-eth/contracts/DstackKms.sol b/dstack/kms/auth-eth/contracts/DstackKms.sol index ca7d1a711..f79325b89 100644 --- a/dstack/kms/auth-eth/contracts/DstackKms.sol +++ b/dstack/kms/auth-eth/contracts/DstackKms.sol @@ -59,6 +59,9 @@ contract DstackKms is Initializable, Ownable2StepUpgradeable, UUPSUpgradeable, E // slither-disable-next-line unindexed-event-address event AppImplementationSet(address implementation); event AppDeployedViaFactory(address indexed appId, address indexed deployer); + /// @notice Additive audit event for reconstructing authorization policy. + /// @dev `value` is the affected bytes32 value, address, or hash of dynamic public data. + event PolicyChanged(address indexed actor, bytes32 indexed policy, bytes32 indexed value, bool enabled); /// @custom:oz-upgrades-unsafe-allow constructor constructor() { @@ -75,6 +78,7 @@ contract DstackKms is Initializable, Ownable2StepUpgradeable, UUPSUpgradeable, E if (_appImplementation != address(0)) { appImplementation = _appImplementation; emit AppImplementationSet(_appImplementation); + _emitPolicy("app-implementation", bytes32(uint256(uint160(_appImplementation))), true); } } @@ -96,12 +100,19 @@ contract DstackKms is Initializable, Ownable2StepUpgradeable, UUPSUpgradeable, E } // Function to authorize upgrades (required by UUPSUpgradeable) - function _authorizeUpgrade(address newImplementation) internal override onlyOwner { } + function _authorizeUpgrade(address newImplementation) internal override onlyOwner { + _emitPolicy("implementation-upgrade", bytes32(uint256(uint160(newImplementation))), true); + } + + function _emitPolicy(string memory policy, bytes32 value, bool enabled) internal { + emit PolicyChanged(msg.sender, keccak256(bytes(policy)), value, enabled); + } // Function to set KMS information function setKmsInfo(KmsInfo memory info) external onlyOwner { kmsInfo = info; emit KmsInfoSet(info.k256Pubkey); + _emitPolicy("kms-info", keccak256(info.k256Pubkey), true); } // Function to set KMS quote @@ -118,6 +129,7 @@ contract DstackKms is Initializable, Ownable2StepUpgradeable, UUPSUpgradeable, E function setGatewayAppId(string memory appId) external onlyOwner { gatewayAppId = appId; emit GatewayAppIdSet(appId); + _emitPolicy("gateway-app-id", keccak256(bytes(appId)), true); } /// @notice Register an app address as known to this KMS. @@ -133,6 +145,7 @@ contract DstackKms is Initializable, Ownable2StepUpgradeable, UUPSUpgradeable, E require(appId != address(0), "Invalid app ID"); registeredApps[appId] = true; emit AppRegistered(appId); + _emitPolicy("registered-app", bytes32(uint256(uint160(appId))), true); } // Function to set DstackApp implementation contract address @@ -140,6 +153,7 @@ contract DstackKms is Initializable, Ownable2StepUpgradeable, UUPSUpgradeable, E require(_implementation != address(0), "Invalid implementation address"); appImplementation = _implementation; emit AppImplementationSet(_implementation); + _emitPolicy("app-implementation", bytes32(uint256(uint160(_implementation))), true); } // Factory method: Deploy and register DstackApp in single transaction @@ -200,36 +214,42 @@ contract DstackKms is Initializable, Ownable2StepUpgradeable, UUPSUpgradeable, E function addKmsAggregatedMr(bytes32 mrAggregated) external onlyOwner { kmsAllowedAggregatedMrs[mrAggregated] = true; emit KmsAggregatedMrAdded(mrAggregated); + _emitPolicy("kms-aggregated-mr", mrAggregated, true); } // Function to deregister an aggregated MR measurement function removeKmsAggregatedMr(bytes32 mrAggregated) external onlyOwner { kmsAllowedAggregatedMrs[mrAggregated] = false; emit KmsAggregatedMrRemoved(mrAggregated); + _emitPolicy("kms-aggregated-mr", mrAggregated, false); } // Function to register a KMS device ID function addKmsDevice(bytes32 deviceId) external onlyOwner { kmsAllowedDeviceIds[deviceId] = true; emit KmsDeviceAdded(deviceId); + _emitPolicy("kms-device", deviceId, true); } // Function to deregister a KMS device ID function removeKmsDevice(bytes32 deviceId) external onlyOwner { kmsAllowedDeviceIds[deviceId] = false; emit KmsDeviceRemoved(deviceId); + _emitPolicy("kms-device", deviceId, false); } // Function to register an image measurement function addOsImageHash(bytes32 osImageHash) external onlyOwner { allowedOsImages[osImageHash] = true; emit OsImageHashAdded(osImageHash); + _emitPolicy("os-image", osImageHash, true); } // Function to deregister an image measurement function removeOsImageHash(bytes32 osImageHash) external onlyOwner { allowedOsImages[osImageHash] = false; emit OsImageHashRemoved(osImageHash); + _emitPolicy("os-image", osImageHash, false); } // Function to check if KMS is allowed to boot diff --git a/dstack/kms/auth-eth/package-lock.json b/dstack/kms/auth-eth/package-lock.json index 22a3794c0..d98dec8b1 100644 --- a/dstack/kms/auth-eth/package-lock.json +++ b/dstack/kms/auth-eth/package-lock.json @@ -9,10 +9,10 @@ "version": "1.0.0", "dependencies": { "@fastify/swagger": "^8.12.0", - "@fastify/swagger-ui": "^2.0.1", + "@fastify/swagger-ui": "^6.1.1", "dotenv": "^16.3.1", "ethers": "^6.17.0", - "fastify": "^5.8.5", + "fastify": "^5.12.1", "yargs": "^17.7.2" }, "devDependencies": { @@ -30,39 +30,25 @@ "integrity": "sha512-nhCBV3quEgesuf7c7KYfperqSS14T8bYuvJ8PcLJp6znkZpFc0AuW4qBtr8eKVyPPe/8RSr7sglCWPU5eaxwKQ==", "license": "MIT" }, - "node_modules/@ampproject/remapping": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", - "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" - }, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/@babel/code-frame": { - "version": "7.26.2", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", - "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-validator-identifier": "^7.25.9", + "@babel/helper-validator-identifier": "^7.29.7", "js-tokens": "^4.0.0", - "picocolors": "^1.0.0" + "picocolors": "^1.1.1" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/compat-data": { - "version": "7.26.3", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.26.3.tgz", - "integrity": "sha512-nHIxvKPniQXpmQLb0vhY3VaFb3S0YrTAwpOWJZh1wn3oJPjJk9Asva204PsBdmAE8vpzfHudT8DB0scYvy9q0g==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", "dev": true, "license": "MIT", "engines": { @@ -70,22 +56,22 @@ } }, "node_modules/@babel/core": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.26.0.tgz", - "integrity": "sha512-i1SLeK+DzNnQ3LL/CswPCa/E5u4lh1k6IAEphON8F+cXt0t9euTshDru0q7/IqMa1PMPz5RnHuHscF8/ZJsStg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@ampproject/remapping": "^2.2.0", - "@babel/code-frame": "^7.26.0", - "@babel/generator": "^7.26.0", - "@babel/helper-compilation-targets": "^7.25.9", - "@babel/helper-module-transforms": "^7.26.0", - "@babel/helpers": "^7.26.0", - "@babel/parser": "^7.26.0", - "@babel/template": "^7.25.9", - "@babel/traverse": "^7.25.9", - "@babel/types": "^7.26.0", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", @@ -111,16 +97,16 @@ } }, "node_modules/@babel/generator": { - "version": "7.26.3", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.3.tgz", - "integrity": "sha512-6FF/urZvD0sTeO7k6/B15pMLC4CHUv1426lzr3N01aHJTl046uCAh9LXW/fzeXXjPNCJ6iABW5XaWOsIZB93aQ==", + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.8.tgz", + "integrity": "sha512-gZbepsdh3WDtgZKWL+vTPh71LSBrm/Y4/QDZBVCcYfmeTEEuoOYwlSy+G1StfJg+/Zy550u/3TATbm7qDbbMtg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/parser": "^7.26.3", - "@babel/types": "^7.26.3", - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.25", + "@babel/parser": "^7.29.8", + "@babel/types": "^7.29.8", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" }, "engines": { @@ -128,14 +114,14 @@ } }, "node_modules/@babel/helper-compilation-targets": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.25.9.tgz", - "integrity": "sha512-j9Db8Suy6yV/VHa4qzrj9yZfZxhLWQdVnRlXxmKLYlhWUVB1sB2G5sxuWYXk/whHD9iW76PmNzxZ4UCnTQTVEQ==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", "dev": true, "license": "MIT", "dependencies": { - "@babel/compat-data": "^7.25.9", - "@babel/helper-validator-option": "^7.25.9", + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" @@ -154,30 +140,40 @@ "semver": "bin/semver.js" } }, + "node_modules/@babel/helper-globals": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/helper-module-imports": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.25.9.tgz", - "integrity": "sha512-tnUA4RsrmflIM6W6RFTLFSXITtl0wKjgpnLgXyowocVPrbYrLUXSBXDgTs8BlbmIzIdlBySRQjINYs2BAkiLtw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", "dev": true, "license": "MIT", "dependencies": { - "@babel/traverse": "^7.25.9", - "@babel/types": "^7.25.9" + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-module-transforms": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.26.0.tgz", - "integrity": "sha512-xO+xu6B5K2czEnQye6BHA7DolFFmS3LB7stHZFaOLb1pAwO1HWLS8fXA+eh0A2yIvltPVmx3eNNDBJA2SLHXFw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-module-imports": "^7.25.9", - "@babel/helper-validator-identifier": "^7.25.9", - "@babel/traverse": "^7.25.9" + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -197,9 +193,9 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", - "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", "dev": true, "license": "MIT", "engines": { @@ -207,9 +203,9 @@ } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", - "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", "dev": true, "license": "MIT", "engines": { @@ -217,9 +213,9 @@ } }, "node_modules/@babel/helper-validator-option": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.25.9.tgz", - "integrity": "sha512-e/zv1co8pp55dNdEcCynfj9X7nyUKUXoUEwfXqaZt0omVOmDe9oOTdKStH4GmAw6zxMFs50ZayuMfHDKlO7Tfw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", "dev": true, "license": "MIT", "engines": { @@ -227,27 +223,27 @@ } }, "node_modules/@babel/helpers": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.0.tgz", - "integrity": "sha512-tbhNuIxNcVb21pInl3ZSjksLCvgdZy9KwJ8brv993QtIVKJBBkYXz4q4ZbAv31GdnC+R90np23L5FbEBlthAEw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/template": "^7.25.9", - "@babel/types": "^7.26.0" + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/parser": { - "version": "7.26.3", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.3.tgz", - "integrity": "sha512-WJ/CvmY8Mea8iDXo6a7RK2wbmJITT5fN3BEkRuFlxVyNx8jOKIIhmC4fSkTcPcf8JyavbBwIe6OpiCOBXt/IcA==", + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.8.tgz", + "integrity": "sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.26.3" + "@babel/types": "^7.29.8" }, "bin": { "parser": "bin/babel-parser.js" @@ -496,48 +492,48 @@ } }, "node_modules/@babel/template": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.9.tgz", - "integrity": "sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.25.9", - "@babel/parser": "^7.25.9", - "@babel/types": "^7.25.9" + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { - "version": "7.26.4", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.26.4.tgz", - "integrity": "sha512-fH+b7Y4p3yqvApJALCPJcwb0/XaOSgtK4pzV6WVjPR5GLFQBRI7pfoX2V2iM48NXvX07NUxxm1Vw98YjqTcU5w==", + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.8.tgz", + "integrity": "sha512-I5z7H3bf/41ktsNVLtpN0wAa336HkqIHQ5BuPLEhTkt1jVSyZpeNKIzTgEWmlxjdg81R0IgUCcaE+Ok3NvrfZg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.26.2", - "@babel/generator": "^7.26.3", - "@babel/parser": "^7.26.3", - "@babel/template": "^7.25.9", - "@babel/types": "^7.26.3", - "debug": "^4.3.1", - "globals": "^11.1.0" + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.8", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.8", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.8", + "debug": "^4.3.1" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/types": { - "version": "7.26.3", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.3.tgz", - "integrity": "sha512-vN5p+1kl59GVKMvTHt55NzzmYVxprfJD+ql7U9NFIfKCBkYE55LYtS+WtPlaYOyzydrKI8Nezd+aZextrd+FMA==", + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", + "integrity": "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^7.25.9", - "@babel/helper-validator-identifier": "^7.25.9" + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -575,13 +571,20 @@ } }, "node_modules/@fastify/accept-negotiator": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@fastify/accept-negotiator/-/accept-negotiator-1.1.0.tgz", - "integrity": "sha512-OIHZrb2ImZ7XG85HXOONLcJWGosv7sIvM2ifAPQVhg9Lv7qdmMBNVaai4QTdyuaqbKM5eO6sLSQOYI7wEQeCJQ==", - "license": "MIT", - "engines": { - "node": ">=14" - } + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@fastify/accept-negotiator/-/accept-negotiator-2.1.0.tgz", + "integrity": "sha512-F3EVbzWt+xcnVaOHmWyIlpuFtbxOln7HDZQsh09MtMmMm/CipMayNt8hnIL8VQi54u2ZociDbf+iluGYkf7B1A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" }, "node_modules/@fastify/ajv-compiler": { "version": "4.0.5", @@ -695,32 +698,69 @@ } }, "node_modules/@fastify/send": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@fastify/send/-/send-2.1.0.tgz", - "integrity": "sha512-yNYiY6sDkexoJR0D8IDy3aRP3+L4wdqCpvx5WP+VtEU58sn7USmKynBzDQex5X42Zzvw2gNzzYgP90UfWShLFA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@fastify/send/-/send-4.1.1.tgz", + "integrity": "sha512-BYo+EiaKwlxH+WetGk6hAs1d39iP0y1gqB8lGF/qwkJ9ZZ/cBY1vx5NvExb9Sc3yRMFjD5X4Eyh4e4+TzRkzdw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], "license": "MIT", "dependencies": { - "@lukeed/ms": "^2.0.1", + "@lukeed/ms": "^2.0.2", "escape-html": "~1.0.3", "fast-decode-uri-component": "^1.0.1", - "http-errors": "2.0.0", - "mime": "^3.0.0" + "http-errors": "^2.0.0", + "mime": "^3" } }, "node_modules/@fastify/static": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@fastify/static/-/static-6.12.0.tgz", - "integrity": "sha512-KK1B84E6QD/FcQWxDI2aiUCwHxMJBI1KeCUzm1BwYpPY1b742+jeKruGHP2uOluuM6OkBPI8CIANrXcCRtC2oQ==", + "version": "10.1.3", + "resolved": "https://registry.npmjs.org/@fastify/static/-/static-10.1.3.tgz", + "integrity": "sha512-W6jqajYS974XjPjB5hQWoxPM8NKM4+p8YmQT6G5IbCa4uhdWSVadZUv75siy1wEA/3ty8RYdpBydfWeu9AqAqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], "license": "MIT", "dependencies": { - "@fastify/accept-negotiator": "^1.0.0", - "@fastify/send": "^2.0.0", - "content-disposition": "^0.5.3", - "fastify-plugin": "^4.0.0", - "glob": "^8.0.1", - "p-limit": "^3.1.0" + "@fastify/accept-negotiator": "^2.0.0", + "@fastify/error": "^4.0.0", + "@fastify/send": "^4.0.0", + "content-disposition": "^2.0.1", + "fastify-plugin": "^6.0.0", + "fastq": "^1.17.1", + "glob": "^13.0.0" } }, + "node_modules/@fastify/static/node_modules/fastify-plugin": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-6.0.0.tgz", + "integrity": "sha512-fZOty7z3O7vOliF6d8bHE3wiEh1KcNnKEQensSgTk9C1DvN6nRLS++XVd86v33Hw/8u9Un8A1zDrQ8ujcQDHEg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, "node_modules/@fastify/swagger": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/@fastify/swagger/-/swagger-8.15.0.tgz", @@ -735,18 +775,44 @@ } }, "node_modules/@fastify/swagger-ui": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@fastify/swagger-ui/-/swagger-ui-2.1.0.tgz", - "integrity": "sha512-mu0C28kMEQDa3miE8f3LmI/OQSmqaKS3dYhZVFO5y4JdgBIPbzZj6COCoRU/P/9nu7UogzzcCJtg89wwLwKtWg==", + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@fastify/swagger-ui/-/swagger-ui-6.1.1.tgz", + "integrity": "sha512-RKCLSHASlzS2JZvHWn14NmEpHyl0yNosGvqzhUumm/LGPG6RWQBf4oscTFt83QDvc5O5Tol3Beup8inAl/k4EA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], "license": "MIT", "dependencies": { - "@fastify/static": "^6.0.0", - "fastify-plugin": "^4.0.0", - "openapi-types": "^12.0.2", - "rfdc": "^1.3.0", - "yaml": "^2.2.2" + "@fastify/static": "^10.1.0", + "fastify-plugin": "^6.0.0", + "openapi-types": "^12.1.3", + "rfdc": "^1.3.1", + "yaml": "^2.4.1" } }, + "node_modules/@fastify/swagger-ui/node_modules/fastify-plugin": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-6.0.0.tgz", + "integrity": "sha512-fZOty7z3O7vOliF6d8bHE3wiEh1KcNnKEQensSgTk9C1DvN6nRLS++XVd86v33Hw/8u9Un8A1zDrQ8ujcQDHEg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -962,10 +1028,11 @@ } }, "node_modules/@jest/reporters/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -994,9 +1061,9 @@ } }, "node_modules/@jest/reporters/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "license": "ISC", "dependencies": { @@ -1112,34 +1179,31 @@ } }, "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.8", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.8.tgz", - "integrity": "sha512-imAbBGkb+ebQyxKgzv5Hu2nmROxoDOXHh80evxdoXNOrvAnVx7zimzc1Oo5h9RlfV4vPXaE2iM5pOFbvOCClWA==", + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", "dev": true, "license": "MIT", "dependencies": { - "@jridgewell/set-array": "^1.2.1", - "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" - }, - "engines": { - "node": ">=6.0.0" } }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", "dev": true, "license": "MIT", - "engines": { - "node": ">=6.0.0" + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" } }, - "node_modules/@jridgewell/set-array": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", - "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", "dev": true, "license": "MIT", "engines": { @@ -1154,9 +1218,9 @@ "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.25", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", - "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", "dev": true, "license": "MIT", "dependencies": { @@ -1693,12 +1757,28 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, "license": "MIT" }, + "node_modules/baseline-browser-mapping": { + "version": "2.11.23", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.23.tgz", + "integrity": "sha512-le521dGVfxM7yRX0EikCoSz+rOK+hHzdDt/E7mG1jOJB/6WAAUuwVroLwaB7ApaUsz5Q0kFlDXLSA9MheUIfRQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", + "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } @@ -1717,9 +1797,9 @@ } }, "node_modules/browserslist": { - "version": "4.24.3", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.3.tgz", - "integrity": "sha512-1CPmv8iobE2fyRMV97dAcMVegvvWKxmq94hkLiAkUGwKVTyDLw33K+ZxiFrREKmmps4rIw6grcCFCnTMSZ/YiA==", + "version": "4.28.9", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.9.tgz", + "integrity": "sha512-EWazOblFYUvlGZcfGhPUPmYh3nikUxBVb+y9MJun5f3hBi812X+8MSQTujLBtgK3cf51fJWbWfOjyeO954d+Eg==", "dev": true, "funding": [ { @@ -1737,10 +1817,11 @@ ], "license": "MIT", "dependencies": { - "caniuse-lite": "^1.0.30001688", - "electron-to-chromium": "^1.5.73", - "node-releases": "^2.0.19", - "update-browserslist-db": "^1.1.1" + "baseline-browser-mapping": "^2.11.20", + "caniuse-lite": "^1.0.30001810", + "electron-to-chromium": "^1.5.420", + "node-releases": "^2.0.54", + "update-browserslist-db": "^1.3.2" }, "bin": { "browserslist": "cli.js" @@ -1800,9 +1881,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001690", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001690.tgz", - "integrity": "sha512-5ExiE3qQN6oF8Clf8ifIDcMRCRE/dMGcETG/XGMD8/XiXm6HXQgQTh1yZYLXXpSOsEUlJm1Xr7kGULZTuGtP/w==", + "version": "1.0.30001810", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", + "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", "dev": true, "funding": [ { @@ -1928,15 +2009,16 @@ "license": "MIT" }, "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-2.0.1.tgz", + "integrity": "sha512-e+H0ZXHSWYrENhQzw1LPuP4oF5MzVKmDU6d3hxlvaPEYLLg62MxtQNPRx4SYSuYJSBUgnQIG4HIN2tEtNv7Dog==", "license": "MIT", - "dependencies": { - "safe-buffer": "5.2.1" - }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/convert-source-map": { @@ -2122,9 +2204,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.78", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.78.tgz", - "integrity": "sha512-UmwIt7HRKN1rsJfddG5UG7rCTCTAKoS9JeOy/R0zSenAyaZ8SU3RuXlwcratxhdxGRNpk03iq8O7BA3W7ibLVw==", + "version": "1.5.427", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.427.tgz", + "integrity": "sha512-n14zb3FdsChZ2BNobqNHAJMcP3ifFv4paox2LvCrfVAQcqGiSURgbJl+PfMpHVCNFkStnNc+RRVtPBTVW5PDgw==", "dev": true, "license": "ISC" }, @@ -2336,9 +2418,9 @@ } }, "node_modules/fast-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz", - "integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==", + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz", + "integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==", "funding": [ { "type": "github", @@ -2352,9 +2434,9 @@ "license": "BSD-3-Clause" }, "node_modules/fastify": { - "version": "5.8.5", - "resolved": "https://registry.npmjs.org/fastify/-/fastify-5.8.5.tgz", - "integrity": "sha512-Yqptv59pQzPgQUSIm87hMqHJmdkb1+GPxdE6vW6FRyVE9G86mt7rOghitiU4JHRaTyDUk9pfeKmDeu70lAwM4Q==", + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/fastify/-/fastify-5.12.1.tgz", + "integrity": "sha512-FWi+tQvwxR/PeRX7Z2mhfEF5ozJ3jn9asiiclzKXNSzJRHAYcU924aIOKAdHFJ+YIKieh3cqr1IwCOvTr41B3Q==", "funding": [ { "type": "github", @@ -2373,11 +2455,11 @@ "@fastify/proxy-addr": "^5.0.0", "abstract-logging": "^2.0.1", "avvio": "^9.0.0", - "fast-json-stringify": "^6.0.0", - "find-my-way": "^9.0.0", + "fast-json-stringify": "^7.0.0", + "find-my-way": "^9.6.0", "light-my-request": "^6.0.0", "pino": "^9.14.0 || ^10.1.0", - "process-warning": "^5.0.0", + "process-warning": "^5.1.0", "rfdc": "^1.3.1", "secure-json-parse": "^4.0.0", "semver": "^7.6.0", @@ -2390,6 +2472,46 @@ "integrity": "sha512-stRHYGeuqpEZTL1Ef0Ovr2ltazUT9g844X5z/zEBFLG8RYlpDiOCIG+ATvYEp+/zmc7sN29mcIMp8gvYplYPIQ==", "license": "MIT" }, + "node_modules/fastify/node_modules/fast-json-stringify": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-7.0.1.tgz", + "integrity": "sha512-eRSayARSbbwlBjpP4vnTTIRD5QPcIrmihPxDeN1DtKnHPg66UuJLx+8hlK1kaFdjvzyQ/dzALoi4vwAQ+T+iZA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "@fastify/merge-json-schemas": "^0.2.0", + "ajv": "^8.12.0", + "ajv-formats": "^3.0.1", + "fast-uri": "^4.0.0", + "json-schema-ref-resolver": "^3.0.0", + "rfdc": "^1.2.0" + } + }, + "node_modules/fastify/node_modules/fast-uri": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-4.1.4.tgz", + "integrity": "sha512-dODXrIxlS9JSdgAnhIUKOosKV1oMtU2VtVw87QRaHzyl5jxO290Ii5tEZfCfzfWNHi3jKWwBSdQj0qIyshdZdQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fastq": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", @@ -2433,9 +2555,9 @@ } }, "node_modules/find-my-way": { - "version": "9.6.0", - "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-9.6.0.tgz", - "integrity": "sha512-Zf4Xve4RymLl7NgaavNebZ01joJ8MfVerOG43wy7SHLO+r+K0C6d/SE0BiR7AV5V1VOCFlOP7ecdo+I4qmiHrQ==", + "version": "9.7.0", + "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-9.7.0.tgz", + "integrity": "sha512-f2JHn75x2JlwUwLenZypgczR7YWMb/uO9BvUXtus+JMgkbIkLADd38cI4EiV+OQqrGo1Zlq6V8wnqMJ8e62wUQ==", "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", @@ -2464,6 +2586,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true, "license": "ISC" }, "node_modules/fsevents": { @@ -2534,33 +2657,56 @@ } }, "node_modules/glob": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", - "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", - "deprecated": "Glob versions prior to v9 are no longer supported", - "license": "ISC", + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "license": "BlueOak-1.0.0", "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^5.0.1", - "once": "^1.3.0" + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" }, "engines": { - "node": ">=12" + "node": "18 || 20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/globals": { - "version": "11.12.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", - "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", - "dev": true, + "node_modules/glob/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", "license": "MIT", "engines": { - "node": ">=4" + "node": "18 || 20 || >=22" + } + }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/graceful-fs": { @@ -2601,19 +2747,23 @@ "license": "MIT" }, "node_modules/http-errors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", - "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", "license": "MIT", "dependencies": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" }, "engines": { "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/human-signals": { @@ -2661,6 +2811,7 @@ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dev": true, "license": "ISC", "dependencies": { "once": "^1.3.0", @@ -2845,19 +2996,20 @@ } }, "node_modules/jake/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "node_modules/jake/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "license": "ISC", "dependencies": { @@ -3022,10 +3174,11 @@ } }, "node_modules/jest-config/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -3054,9 +3207,9 @@ } }, "node_modules/jest-config/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "license": "ISC", "dependencies": { @@ -3363,10 +3516,11 @@ } }, "node_modules/jest-runtime/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -3395,9 +3549,9 @@ } }, "node_modules/jest-runtime/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "license": "ISC", "dependencies": { @@ -3548,9 +3702,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "3.15.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.2.tgz", + "integrity": "sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==", "dev": true, "license": "MIT", "dependencies": { @@ -3807,9 +3961,10 @@ } }, "node_modules/minimatch": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", - "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" @@ -3818,6 +3973,15 @@ "node": ">=10" } }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -3839,11 +4003,14 @@ "license": "MIT" }, "node_modules/node-releases": { - "version": "2.0.19", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", - "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==", + "version": "2.0.55", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.55.tgz", + "integrity": "sha512-mIrE/Cw9y+9Au6dS5vDKDhQza9YvG6w+ZrS6X+ZzA7yFW/soAeaups4Qzn1bL6g5FVy8WtP79+0j82oPIbqRjQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=18" + } }, "node_modules/normalize-path": { "version": "3.0.0", @@ -3881,6 +4048,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -3912,6 +4080,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, "license": "MIT", "dependencies": { "yocto-queue": "^0.1.0" @@ -4018,6 +4187,31 @@ "dev": true, "license": "MIT" }, + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "11.5.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", + "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -4127,9 +4321,9 @@ } }, "node_modules/process-warning": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", - "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.1.0.tgz", + "integrity": "sha512-jQSaVHsPgtyw60e1rQ/A+/ArPEj/S8pS/vFnyGa/gYFXrKk/6RuDkoqVDQ5NI5MmS01698ltlAk0NoDBNLujRw==", "funding": [ { "type": "github", @@ -4301,26 +4495,6 @@ "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==", "license": "MIT" }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, "node_modules/safe-regex2": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-regex2/-/safe-regex2-5.1.1.tgz", @@ -4499,9 +4673,9 @@ } }, "node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", "license": "MIT", "engines": { "node": ">= 0.8" @@ -4622,10 +4796,11 @@ } }, "node_modules/test-exclude/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -4654,9 +4829,9 @@ } }, "node_modules/test-exclude/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "license": "ISC", "dependencies": { @@ -4865,9 +5040,9 @@ "license": "MIT" }, "node_modules/update-browserslist-db": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.1.tgz", - "integrity": "sha512-R8UzCaa9Az+38REPiJ1tXlImTJXlVfgHZsglwBD/k6nj76ctsH1E3q4doGrukiLQd3sGQYu56r5+lo5r94l29A==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.3.tgz", + "integrity": "sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ==", "dev": true, "funding": [ { @@ -4886,7 +5061,7 @@ "license": "MIT", "dependencies": { "escalade": "^3.2.0", - "picocolors": "^1.1.0" + "picocolors": "^1.1.1" }, "bin": { "update-browserslist-db": "cli.js" @@ -4973,6 +5148,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, "license": "ISC" }, "node_modules/write-file-atomic": { @@ -5082,6 +5258,7 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, "license": "MIT", "engines": { "node": ">=10" diff --git a/dstack/kms/auth-eth/package.json b/dstack/kms/auth-eth/package.json index c44a9a1b9..0a78f6830 100644 --- a/dstack/kms/auth-eth/package.json +++ b/dstack/kms/auth-eth/package.json @@ -19,10 +19,10 @@ }, "dependencies": { "@fastify/swagger": "^8.12.0", - "@fastify/swagger-ui": "^2.0.1", + "@fastify/swagger-ui": "^6.1.1", "dotenv": "^16.3.1", "ethers": "^6.17.0", - "fastify": "^5.8.5", + "fastify": "^5.12.1", "yargs": "^17.7.2" }, "devDependencies": { diff --git a/dstack/kms/auth-eth/script/Upgrade.s.sol b/dstack/kms/auth-eth/script/Upgrade.s.sol index 82461a115..94562b05f 100644 --- a/dstack/kms/auth-eth/script/Upgrade.s.sol +++ b/dstack/kms/auth-eth/script/Upgrade.s.sol @@ -58,7 +58,7 @@ contract UpgradeKmsToV2 is Script { vm.startBroadcast(); // Upgrade to a specific contract version - Upgrades.upgradeProxy(kmsProxy, "contracts/test-utils/DstackKmsV2.sol:DstackKmsV2", ""); + Upgrades.upgradeProxy(kmsProxy, "DstackKmsV2.sol:DstackKmsV2", ""); vm.stopBroadcast(); @@ -76,7 +76,7 @@ contract UpgradeAppToV2 is Script { vm.startBroadcast(); // Upgrade to a specific contract version - Upgrades.upgradeProxy(appProxy, "contracts/test-utils/DstackAppV2.sol:DstackAppV2", ""); + Upgrades.upgradeProxy(appProxy, "DstackAppV2.sol:DstackAppV2", ""); vm.stopBroadcast(); diff --git a/dstack/kms/auth-eth/src/main.test.ts b/dstack/kms/auth-eth/src/main.test.ts index 84b7fce07..842f1e36b 100644 --- a/dstack/kms/auth-eth/src/main.test.ts +++ b/dstack/kms/auth-eth/src/main.test.ts @@ -60,6 +60,20 @@ describe('Server', () => { expect(mockCheckBoot).toHaveBeenCalledWith(mockBootInfo, false); }); + it('should reject oversized and non-hex measurements before backend use', async () => { + jest.clearAllMocks(); + for (const mrAggregated of ['0x' + 'ab'.repeat(33), 'not-hex']) { + const response = await app.inject({ + method: 'POST', + url: '/bootAuth/app', + payload: { ...mockBootInfo, mrAggregated } + }); + expect(response.statusCode).toBe(400); + expect(JSON.parse(response.payload)).toEqual({ isAllowed: false, reason: 'invalid authorization request', gatewayAppId: '' }); + } + expect(app.ethereum.checkBoot).not.toHaveBeenCalled(); + }); + it('should return 400 for invalid boot info', async () => { const response = await app.inject({ method: 'POST', @@ -111,7 +125,7 @@ describe('Server', () => { expect(response.statusCode).toBe(200); const result = JSON.parse(response.payload); expect(result.isAllowed).toBe(false); - expect(result.reason).toMatch(/Test backend error/); + expect(result.reason).toBe('authorization backend unavailable'); }); }); }); diff --git a/dstack/kms/auth-eth/src/server.ts b/dstack/kms/auth-eth/src/server.ts index d2db629b9..54dd14346 100644 --- a/dstack/kms/auth-eth/src/server.ts +++ b/dstack/kms/auth-eth/src/server.ts @@ -19,17 +19,26 @@ export async function build(): Promise<FastifyInstance> { }); // Register schema for request/response validation + const hex = (bytes: number, description: string) => ({ + type: 'string', + pattern: `^(?:0x[0-9a-fA-F]{0,${bytes * 2}}|[0-9a-fA-F]{0,${bytes * 2}})$`, + description, + }); + server.addSchema({ $id: 'bootInfo', type: 'object', required: ['mrAggregated', 'osImageHash', 'appId', 'composeHash', 'instanceId', 'deviceId'], properties: { - mrAggregated: { type: 'string', description: 'Aggregated MR measurement' }, - osImageHash: { type: 'string', description: 'OS Image hash' }, - appId: { type: 'string', description: 'Application ID' }, - composeHash: { type: 'string', description: 'Compose hash' }, - instanceId: { type: 'string', description: 'Instance ID' }, - deviceId: { type: 'string', description: 'Device ID' } + mrAggregated: hex(32, 'Aggregated MR measurement'), + osImageHash: hex(32, 'OS Image hash'), + appId: hex(20, 'Application ID'), + composeHash: hex(32, 'Compose hash'), + instanceId: hex(20, 'Instance ID'), + deviceId: hex(32, 'Device ID'), + tcbStatus: { type: 'string', maxLength: 128, default: '' }, + advisoryIds: { type: 'array', maxItems: 128, items: { type: 'string', maxLength: 256 }, default: [] }, + mrSystem: { ...hex(32, 'System MR measurement'), default: '' }, } }); @@ -50,21 +59,45 @@ export async function build(): Promise<FastifyInstance> { const provider = new ethers.JsonRpcProvider(rpcUrl); server.decorate('ethereum', new EthereumBackend(provider, kmsContractAddr)); - server.get('/', async (request, reply) => { - const batch = await Promise.all([ - server.ethereum.getGatewayAppId(), - server.ethereum.getChainId(), - server.ethereum.getAppImplementation(), - ]); - return { - status: 'ok', - kmsContractAddr: kmsContractAddr, - ethRpcUrl: rpcUrl, - gatewayAppId: batch[0], - chainId: batch[1], - appAuthImplementation: batch[2], // NOTE: for backward compatibility - appImplementation: batch[2], - }; + const publicRpcEndpoint = (value: string): string => { + try { + const endpoint = new URL(value); + return `${endpoint.protocol}//${endpoint.host}`; + } catch { + return 'configured'; + } + }; + const backendUnavailable = 'authorization backend unavailable'; + const invalidRequest = { isAllowed: false, reason: 'invalid authorization request', gatewayAppId: '' }; + + server.setErrorHandler((error, request, reply) => { + if (typeof error === 'object' && error !== null && 'validation' in error) { + return reply.code(400).send(invalidRequest); + } + request.log.error('authorization request failed'); + return reply.code(500).send({ status: 'error', message: backendUnavailable }); + }); + + server.get('/', async (_request, reply) => { + try { + const batch = await Promise.all([ + server.ethereum.getGatewayAppId(), + server.ethereum.getChainId(), + server.ethereum.getAppImplementation(), + ]); + return { + status: 'ok', + kmsContractAddr: kmsContractAddr, + ethRpcUrl: publicRpcEndpoint(rpcUrl), + gatewayAppId: batch[0], + chainId: batch[1], + appAuthImplementation: batch[2], // NOTE: for backward compatibility + appImplementation: batch[2], + }; + } catch { + _request.log.error('authorization backend health check failed'); + return reply.code(500).send({ status: 'error', message: backendUnavailable }); + } }); // Define routes @@ -82,11 +115,11 @@ export async function build(): Promise<FastifyInstance> { try { return await server.ethereum.checkBoot(request.body, false); } catch (error) { - console.error(error); + request.log.error('application authorization backend failed'); reply.code(200).send({ isAllowed: false, gatewayAppId: '', - reason: `${error instanceof Error ? error.message : String(error)}` + reason: backendUnavailable }); } }); @@ -106,12 +139,12 @@ export async function build(): Promise<FastifyInstance> { return await server.ethereum.checkBoot(request.body, true); } catch (error) { if (!(error instanceof Error && "Test backend error" == error.message)) { - console.error(error); + request.log.error('KMS authorization backend failed'); } reply.code(200).send({ isAllowed: false, gatewayAppId: '', - reason: `${error instanceof Error ? error.message : String(error)}` + reason: backendUnavailable }); } }); diff --git a/dstack/kms/auth-eth/test/EventAudit.t.sol b/dstack/kms/auth-eth/test/EventAudit.t.sol new file mode 100644 index 000000000..9631b355a --- /dev/null +++ b/dstack/kms/auth-eth/test/EventAudit.t.sol @@ -0,0 +1,186 @@ +/* + * SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> + * + * SPDX-License-Identifier: Apache-2.0 + */ + +pragma solidity ^0.8.24; + +import "forge-std/Test.sol"; +import "openzeppelin-foundry-upgrades/Upgrades.sol"; +import "../contracts/DstackKms.sol"; +import "../contracts/DstackApp.sol"; + +contract EventAuditTest is Test { + bytes32 private constant AUDIT_TOPIC = keccak256("PolicyChanged(address,bytes32,bytes32,bool)"); + + address private owner; + address private outsider; + DstackKms private kms; + DstackApp private app; + + function setUp() public { + owner = makeAddr("audit-owner"); + outsider = makeAddr("audit-outsider"); + vm.startPrank(owner); + DstackApp appImplementation = new DstackApp(); + kms = DstackKms( + Upgrades.deployUUPSProxy( + "DstackKms.sol", abi.encodeCall(DstackKms.initialize, (owner, address(appImplementation))) + ) + ); + app = DstackApp( + Upgrades.deployUUPSProxy( + "DstackApp.sol", + abi.encodeWithSignature( + "initialize(address,bool,bool,bool,bytes32,bytes32)", + owner, + false, + false, + false, + bytes32(0), + bytes32(0) + ) + ) + ); + vm.stopPrank(); + } + + function test_KmsPolicyEventsReconstructQueriedState() public { + bytes32 mr = keccak256("audit-mr"); + bytes32 device = keccak256("audit-device"); + bytes32 image = keccak256("audit-image"); + string memory gateway = "audit-gateway"; + + vm.recordLogs(); + vm.startPrank(owner); + kms.addKmsAggregatedMr(mr); + kms.addKmsDevice(device); + kms.addOsImageHash(image); + kms.setGatewayAppId(gateway); + kms.registerApp(address(app)); + kms.removeKmsAggregatedMr(mr); + vm.stopPrank(); + Vm.Log[] memory logs = vm.getRecordedLogs(); + + _assertAudit(logs, owner, "kms-aggregated-mr", mr, true); + _assertAudit(logs, owner, "kms-device", device, true); + _assertAudit(logs, owner, "os-image", image, true); + _assertAudit(logs, owner, "gateway-app-id", keccak256(bytes(gateway)), true); + _assertAudit(logs, owner, "registered-app", bytes32(uint256(uint160(address(app)))), true); + _assertAudit(logs, owner, "kms-aggregated-mr", mr, false); + + assertFalse(kms.kmsAllowedAggregatedMrs(mr)); + assertTrue(kms.kmsAllowedDeviceIds(device)); + assertTrue(kms.allowedOsImages(image)); + assertEq(kms.gatewayAppId(), gateway); + assertTrue(kms.registeredApps(address(app))); + } + + function test_AppPolicyEventsReconstructQueriedState() public { + bytes32 composeHash = keccak256("audit-compose"); + bytes32 device = keccak256("audit-app-device"); + + vm.recordLogs(); + vm.startPrank(owner); + app.addComposeHash(composeHash); + app.addDevice(device); + app.setAllowAnyDevice(true); + app.setRequireTcbUpToDate(true); + app.removeDevice(device); + app.disableUpgrades(); + vm.stopPrank(); + Vm.Log[] memory logs = vm.getRecordedLogs(); + + _assertAudit(logs, owner, "compose-hash", composeHash, true); + _assertAudit(logs, owner, "device", device, true); + _assertAudit(logs, owner, "allow-any-device", bytes32(0), true); + _assertAudit(logs, owner, "require-tcb-up-to-date", bytes32(0), true); + _assertAudit(logs, owner, "device", device, false); + _assertAudit(logs, owner, "upgrades-disabled", bytes32(0), true); + + assertTrue(app.allowedComposeHashes(composeHash)); + assertFalse(app.allowedDeviceIds(device)); + assertTrue(app.allowAnyDevice()); + assertTrue(app.requireTcbUpToDate()); + } + + function test_InvalidMutationEmitsNoAuditAndLeavesNoPartialState() public { + bytes32 image = keccak256("unauthorized-image"); + vm.recordLogs(); + vm.prank(outsider); + vm.expectRevert(); + kms.addOsImageHash(image); + Vm.Log[] memory logs = vm.getRecordedLogs(); + assertEq(_auditCount(logs), 0); + assertFalse(kms.allowedOsImages(image)); + } + + function test_ReorgDropsOrphanEventAndCanonicalEventRebuildsState() public { + bytes32 orphaned = keccak256("orphaned-compose"); + bytes32 canonical = keccak256("canonical-compose"); + uint256 snapshot = vm.snapshotState(); + + vm.recordLogs(); + vm.prank(owner); + app.addComposeHash(orphaned); + Vm.Log[] memory orphanLogs = vm.getRecordedLogs(); + _assertAudit(orphanLogs, owner, "compose-hash", orphaned, true); + assertTrue(app.allowedComposeHashes(orphaned)); + + assertTrue(vm.revertToState(snapshot)); + assertFalse(app.allowedComposeHashes(orphaned)); + + vm.recordLogs(); + vm.prank(owner); + app.addComposeHash(canonical); + Vm.Log[] memory canonicalLogs = vm.getRecordedLogs(); + _assertAudit(canonicalLogs, owner, "compose-hash", canonical, true); + assertEq(_auditCount(canonicalLogs), 1); + assertTrue(app.allowedComposeHashes(canonical)); + assertFalse(app.allowedComposeHashes(orphaned)); + } + + function test_UpgradeAuditIncludesActorAndImplementation() public { + vm.startPrank(owner); + DstackApp replacement = new DstackApp(); + vm.recordLogs(); + app.upgradeToAndCall(address(replacement), ""); + Vm.Log[] memory logs = vm.getRecordedLogs(); + vm.stopPrank(); + + _assertAudit(logs, owner, "implementation-upgrade", bytes32(uint256(uint160(address(replacement)))), true); + assertEq(Upgrades.getImplementationAddress(address(app)), address(replacement)); + } + + function _assertAudit( + Vm.Log[] memory logs, + address actor, + string memory policy, + bytes32 value, + bool enabled + ) + private + pure + { + bytes32 actorTopic = bytes32(uint256(uint160(actor))); + bytes32 policyTopic = keccak256(bytes(policy)); + for (uint256 i = 0; i < logs.length; ++i) { + Vm.Log memory entry = logs[i]; + if ( + entry.topics.length == 4 && entry.topics[0] == AUDIT_TOPIC && entry.topics[1] == actorTopic + && entry.topics[2] == policyTopic && entry.topics[3] == value + && abi.decode(entry.data, (bool)) == enabled + ) { + return; + } + } + revert("expected policy audit event not found"); + } + + function _auditCount(Vm.Log[] memory logs) private pure returns (uint256 count) { + for (uint256 i = 0; i < logs.length; ++i) { + if (logs[i].topics.length == 4 && logs[i].topics[0] == AUDIT_TOPIC) ++count; + } + } +} diff --git a/dstack/kms/auth-eth/tsconfig.json b/dstack/kms/auth-eth/tsconfig.json index 467ee2211..fc23863ce 100644 --- a/dstack/kms/auth-eth/tsconfig.json +++ b/dstack/kms/auth-eth/tsconfig.json @@ -2,17 +2,26 @@ "compilerOptions": { "target": "ES2020", "module": "commonjs", - "lib": ["es2020"], + "lib": [ + "es2020" + ], "strict": true, "esModuleInterop": true, "skipLibCheck": true, "forceConsistentCasingInFileNames": true, "outDir": "./dist", - "rootDir": ".", + "rootDir": "./src", "resolveJsonModule": true, - "types": ["node", "jest"], + "types": [ + "node", + "jest" + ], "baseUrl": "." }, - "include": ["src/**/*"], - "exclude": ["node_modules"] + "include": [ + "src/**/*" + ], + "exclude": [ + "node_modules" + ] } diff --git a/dstack/kms/auth-mock/Dockerfile b/dstack/kms/auth-mock/Dockerfile index 3bb12a25d..75207418a 100644 --- a/dstack/kms/auth-mock/Dockerfile +++ b/dstack/kms/auth-mock/Dockerfile @@ -6,7 +6,7 @@ FROM oven/bun:1-alpine WORKDIR /app ARG DSTACK_REV -ARG DSTACK_BRANCH=master +ARG DSTACK_BRANCH=next RUN apk add --no-cache git RUN git clone --branch ${DSTACK_BRANCH} https://github.com/Dstack-TEE/dstack.git && \ diff --git a/dstack/kms/auth-mock/bun.lock b/dstack/kms/auth-mock/bun.lock index b525e9fbd..35fa3ece6 100644 --- a/dstack/kms/auth-mock/bun.lock +++ b/dstack/kms/auth-mock/bun.lock @@ -6,7 +6,7 @@ "name": "auth-mock", "dependencies": { "@hono/zod-validator": "0.2.2", - "hono": "4.12.7", + "hono": "4.12.34", "zod": "3.25.76", }, "devDependencies": { @@ -16,62 +16,24 @@ "openapi-types": "12.1.3", "oxlint": "0.9.10", "typescript": "5.8.3", - "vitest": "1.6.1", + "vitest": "4.1.0", }, }, }, "packages": { - "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.21.5", "", { "os": "aix", "cpu": "ppc64" }, "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ=="], + "@emnapi/core": ["@emnapi/core@1.11.1", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" } }, "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ=="], - "@esbuild/android-arm": ["@esbuild/android-arm@0.21.5", "", { "os": "android", "cpu": "arm" }, "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg=="], + "@emnapi/runtime": ["@emnapi/runtime@1.11.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw=="], - "@esbuild/android-arm64": ["@esbuild/android-arm64@0.21.5", "", { "os": "android", "cpu": "arm64" }, "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A=="], - - "@esbuild/android-x64": ["@esbuild/android-x64@0.21.5", "", { "os": "android", "cpu": "x64" }, "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA=="], - - "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.21.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ=="], - - "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.21.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw=="], - - "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.21.5", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g=="], - - "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.21.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ=="], - - "@esbuild/linux-arm": ["@esbuild/linux-arm@0.21.5", "", { "os": "linux", "cpu": "arm" }, "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA=="], - - "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.21.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q=="], - - "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.21.5", "", { "os": "linux", "cpu": "ia32" }, "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg=="], - - "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg=="], - - "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg=="], - - "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.21.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w=="], - - "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA=="], - - "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.21.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A=="], - - "@esbuild/linux-x64": ["@esbuild/linux-x64@0.21.5", "", { "os": "linux", "cpu": "x64" }, "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ=="], - - "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.21.5", "", { "os": "none", "cpu": "x64" }, "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg=="], - - "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.21.5", "", { "os": "openbsd", "cpu": "x64" }, "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow=="], - - "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.21.5", "", { "os": "sunos", "cpu": "x64" }, "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg=="], - - "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.21.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A=="], - - "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.21.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA=="], - - "@esbuild/win32-x64": ["@esbuild/win32-x64@0.21.5", "", { "os": "win32", "cpu": "x64" }, "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw=="], + "@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.2.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA=="], "@hono/zod-validator": ["@hono/zod-validator@0.2.2", "", { "peerDependencies": { "hono": ">=3.9.0", "zod": "^3.19.1" } }, "sha512-dSDxaPV70Py8wuIU2QNpoVEIOSzSXZ/6/B/h4xA7eOMz7+AarKTSGV8E6QwrdcCbBLkpqfJ4Q2TmBO0eP1tCBQ=="], "@jest/schemas": ["@jest/schemas@29.6.3", "", { "dependencies": { "@sinclair/typebox": "^0.27.8" } }, "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA=="], - "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.4", "", {}, "sha512-VT2+G1VQs/9oz078bLrYbecdZKs912zQlkelYpuf+SXF+QvZDYJlbx/LSx+meSAwdDFnF8FVXW92AVjjkVmgFw=="], + "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], + + "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.6", "", { "dependencies": { "@tybys/wasm-util": "^0.10.3" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg=="], "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], @@ -79,6 +41,8 @@ "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], + "@oxc-project/types": ["@oxc-project/types@0.139.0", "", {}, "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw=="], + "@oxlint/darwin-arm64": ["@oxlint/darwin-arm64@0.9.10", "", { "os": "darwin", "cpu": "arm64" }, "sha512-eOXKZYq5bnCSgDefgM5bzAg+4Fc//Rc4yjgKN8iDWUARweCaChiQXb6TXX8MfEfs6qayEMy6yVj0pqoFz0B1aw=="], "@oxlint/darwin-x64": ["@oxlint/darwin-x64@0.9.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-UeYICDvLUaUOcY+0ugZUEmBMRLP+x8iTgL7TeY6BlpGw2ahbtUOTbyIIRWtr/0O++TnjZ+v8TzhJ9crw6Ij6dg=="], @@ -97,49 +61,49 @@ "@polka/url": ["@polka/url@1.0.0-next.29", "", {}, "sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww=="], - "@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.45.1", "", { "os": "android", "cpu": "arm" }, "sha512-NEySIFvMY0ZQO+utJkgoMiCAjMrGvnbDLHvcmlA33UXJpYBCvlBEbMMtV837uCkS+plG2umfhn0T5mMAxGrlRA=="], + "@rolldown/binding-android-arm64": ["@rolldown/binding-android-arm64@1.1.5", "", { "os": "android", "cpu": "arm64" }, "sha512-lZg8fqIv2v7FF237bwMgzGZEJvGL79/s5knJ/i6FmsGF4XXlzccZ4jb+TrFIxtSSxFtIpdsgrPZeMk1I9AFcyQ=="], - "@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.45.1", "", { "os": "android", "cpu": "arm64" }, "sha512-ujQ+sMXJkg4LRJaYreaVx7Z/VMgBBd89wGS4qMrdtfUFZ+TSY5Rs9asgjitLwzeIbhwdEhyj29zhst3L1lKsRQ=="], + "@rolldown/binding-darwin-arm64": ["@rolldown/binding-darwin-arm64@1.1.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-51Bnx9pNiMRKSUNtBfySkNJ9vMU9Hh3I1ozDd6gyPPYzaXCfnptUcEZxXGYFn+ul2dtcMUiqGR1Yai2K10uoTw=="], - "@rollup/rollup-darwin-arm64": ["@rollup/rollup-darwin-arm64@4.45.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-FSncqHvqTm3lC6Y13xncsdOYfxGSLnP+73k815EfNmpewPs+EyM49haPS105Rh4aF5mJKywk9X0ogzLXZzN9lA=="], + "@rolldown/binding-darwin-x64": ["@rolldown/binding-darwin-x64@1.1.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-Tm+gbfC0aHu1tBA/JvKQh32S0K6YgCHkiAF4/W6xX0K0RmNuc94VeK419dJoE65R5aRxmo+noZQSWrAMF6yb6g=="], - "@rollup/rollup-darwin-x64": ["@rollup/rollup-darwin-x64@4.45.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-2/vVn/husP5XI7Fsf/RlhDaQJ7x9zjvC81anIVbr4b/f0xtSmXQTFcGIQ/B1cXIYM6h2nAhJkdMHTnD7OtQ9Og=="], + "@rolldown/binding-freebsd-x64": ["@rolldown/binding-freebsd-x64@1.1.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-JMzDKCCXq93YccG5gz3hvOs1oXRKAf0XYpfOS88e+wZrC8Iugj6j68867vrYZkvpDDpKn/KoKORThmchMpF6TA=="], - "@rollup/rollup-freebsd-arm64": ["@rollup/rollup-freebsd-arm64@4.45.1", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-4g1kaDxQItZsrkVTdYQ0bxu4ZIQ32cotoQbmsAnW1jAE4XCMbcBPDirX5fyUzdhVCKgPcrwWuucI8yrVRBw2+g=="], + "@rolldown/binding-linux-arm-gnueabihf": ["@rolldown/binding-linux-arm-gnueabihf@1.1.5", "", { "os": "linux", "cpu": "arm" }, "sha512-uML21j2K5TfPGutKxub+M+nLjZIrWjXQ5Grx4lCe/nimTj9B4L63zHpjXLl4y0L3mcm2htEQIb06oCG/szerNw=="], - "@rollup/rollup-freebsd-x64": ["@rollup/rollup-freebsd-x64@4.45.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-L/6JsfiL74i3uK1Ti2ZFSNsp5NMiM4/kbbGEcOCps99aZx3g8SJMO1/9Y0n/qKlWZfn6sScf98lEOUe2mBvW9A=="], + "@rolldown/binding-linux-arm64-gnu": ["@rolldown/binding-linux-arm64-gnu@1.1.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-navSiuTMogvnQoZoM/v+l3ZWo50/NTwSHSzheABx/RCnmUPaKwq9qSo4Br2OYRs21+Fz8uFqITZM3H4opOB0/Q=="], - "@rollup/rollup-linux-arm-gnueabihf": ["@rollup/rollup-linux-arm-gnueabihf@4.45.1", "", { "os": "linux", "cpu": "arm" }, "sha512-RkdOTu2jK7brlu+ZwjMIZfdV2sSYHK2qR08FUWcIoqJC2eywHbXr0L8T/pONFwkGukQqERDheaGTeedG+rra6Q=="], + "@rolldown/binding-linux-arm64-musl": ["@rolldown/binding-linux-arm64-musl@1.1.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-lAryqH7IteztmCXQXk0etKj4wBQ7Gx5S6LjKhsgp9zb8I5bsuvU/2llH1hDQcjsFeqIsovMVN339/8pUDDBXxA=="], - "@rollup/rollup-linux-arm-musleabihf": ["@rollup/rollup-linux-arm-musleabihf@4.45.1", "", { "os": "linux", "cpu": "arm" }, "sha512-3kJ8pgfBt6CIIr1o+HQA7OZ9mp/zDk3ctekGl9qn/pRBgrRgfwiffaUmqioUGN9hv0OHv2gxmvdKOkARCtRb8Q=="], + "@rolldown/binding-linux-ppc64-gnu": ["@rolldown/binding-linux-ppc64-gnu@1.1.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-fsK/sNBnxzBlL4O1JNrZakVQxPspqpED5dLtNsZS9oOKmtSpdNIzxH2kkol5HYTWJN47sE20ztMJPxfZ89qGOg=="], - "@rollup/rollup-linux-arm64-gnu": ["@rollup/rollup-linux-arm64-gnu@4.45.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-k3dOKCfIVixWjG7OXTCOmDfJj3vbdhN0QYEqB+OuGArOChek22hn7Uy5A/gTDNAcCy5v2YcXRJ/Qcnm4/ma1xw=="], + "@rolldown/binding-linux-s390x-gnu": ["@rolldown/binding-linux-s390x-gnu@1.1.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-gLYb4BIadlfTOYT5gO503n8zQjXflgzpD0FcyKh0Mzx3rqCZKnHoJWV9xe1KXUJ5lx2JfcSHr/mhzS0PC/McAA=="], - "@rollup/rollup-linux-arm64-musl": ["@rollup/rollup-linux-arm64-musl@4.45.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-PmI1vxQetnM58ZmDFl9/Uk2lpBBby6B6rF4muJc65uZbxCs0EA7hhKCk2PKlmZKuyVSHAyIw3+/SiuMLxKxWog=="], + "@rolldown/binding-linux-x64-gnu": ["@rolldown/binding-linux-x64-gnu@1.1.5", "", { "os": "linux", "cpu": "x64" }, "sha512-FjcpEKUyJygHgs1o50VYNvkt5+7Le/VEdYt0AkRpkL33MnyQfwr8l5mXwMmfmTbyMPr5vJLC+8/Gd9gXnwU1QQ=="], - "@rollup/rollup-linux-loongarch64-gnu": ["@rollup/rollup-linux-loongarch64-gnu@4.45.1", "", { "os": "linux", "cpu": "none" }, "sha512-9UmI0VzGmNJ28ibHW2GpE2nF0PBQqsyiS4kcJ5vK+wuwGnV5RlqdczVocDSUfGX/Na7/XINRVoUgJyFIgipoRg=="], + "@rolldown/binding-linux-x64-musl": ["@rolldown/binding-linux-x64-musl@1.1.5", "", { "os": "linux", "cpu": "x64" }, "sha512-Me+PfPI2TMeOQk0gYWfLQZtTktrmzbr8cDboqX83XKc7UrgAi55gF+2dUkWdxd19n55Essp2yeca+O9N5rBxHg=="], - "@rollup/rollup-linux-powerpc64le-gnu": ["@rollup/rollup-linux-powerpc64le-gnu@4.45.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-7nR2KY8oEOUTD3pBAxIBBbZr0U7U+R9HDTPNy+5nVVHDXI4ikYniH1oxQz9VoB5PbBU1CZuDGHkLJkd3zLMWsg=="], + "@rolldown/binding-openharmony-arm64": ["@rolldown/binding-openharmony-arm64@1.1.5", "", { "os": "none", "cpu": "arm64" }, "sha512-yc5WrLzXks6zCQfn9Oxr8pORKyl/pF+QjHmW/Qx3qu0oyrrNC+y2JLTU1E2rcWYAmzlnqngWXHQjy51VzW70Vw=="], - "@rollup/rollup-linux-riscv64-gnu": ["@rollup/rollup-linux-riscv64-gnu@4.45.1", "", { "os": "linux", "cpu": "none" }, "sha512-nlcl3jgUultKROfZijKjRQLUu9Ma0PeNv/VFHkZiKbXTBQXhpytS8CIj5/NfBeECZtY2FJQubm6ltIxm/ftxpw=="], + "@rolldown/binding-wasm32-wasi": ["@rolldown/binding-wasm32-wasi@1.1.5", "", { "dependencies": { "@emnapi/core": "1.11.1", "@emnapi/runtime": "1.11.1", "@napi-rs/wasm-runtime": "^1.1.6" }, "cpu": "none" }, "sha512-VbQGPX2b4r48TAMIM2cjgluIM1HYutm4pcTEJsle7iEP7sB1dFqtPLBVbdLAZCxy1txCcPxf4QFf4v8uvltPqA=="], - "@rollup/rollup-linux-riscv64-musl": ["@rollup/rollup-linux-riscv64-musl@4.45.1", "", { "os": "linux", "cpu": "none" }, "sha512-HJV65KLS51rW0VY6rvZkiieiBnurSzpzore1bMKAhunQiECPuxsROvyeaot/tcK3A3aGnI+qTHqisrpSgQrpgA=="], + "@rolldown/binding-win32-arm64-msvc": ["@rolldown/binding-win32-arm64-msvc@1.1.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-gHv82k63z4qpV5+Q1y/12KrK0ltWBukVDI8nZcbT7Tt/ZlOIVwppazneq0F93oDxTo3IgAMEDIoQh3E2n6mVsw=="], - "@rollup/rollup-linux-s390x-gnu": ["@rollup/rollup-linux-s390x-gnu@4.45.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-NITBOCv3Qqc6hhwFt7jLV78VEO/il4YcBzoMGGNxznLgRQf43VQDae0aAzKiBeEPIxnDrACiMgbqjuihx08OOw=="], + "@rolldown/binding-win32-x64-msvc": ["@rolldown/binding-win32-x64-msvc@1.1.5", "", { "os": "win32", "cpu": "x64" }, "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA=="], - "@rollup/rollup-linux-x64-gnu": ["@rollup/rollup-linux-x64-gnu@4.45.1", "", { "os": "linux", "cpu": "x64" }, "sha512-+E/lYl6qu1zqgPEnTrs4WysQtvc/Sh4fC2nByfFExqgYrqkKWp1tWIbe+ELhixnenSpBbLXNi6vbEEJ8M7fiHw=="], + "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.1", "", {}, "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw=="], - "@rollup/rollup-linux-x64-musl": ["@rollup/rollup-linux-x64-musl@4.45.1", "", { "os": "linux", "cpu": "x64" }, "sha512-a6WIAp89p3kpNoYStITT9RbTbTnqarU7D8N8F2CV+4Cl9fwCOZraLVuVFvlpsW0SbIiYtEnhCZBPLoNdRkjQFw=="], + "@sinclair/typebox": ["@sinclair/typebox@0.27.8", "", {}, "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA=="], - "@rollup/rollup-win32-arm64-msvc": ["@rollup/rollup-win32-arm64-msvc@4.45.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-T5Bi/NS3fQiJeYdGvRpTAP5P02kqSOpqiopwhj0uaXB6nzs5JVi2XMJb18JUSKhCOX8+UE1UKQufyD6Or48dJg=="], + "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], - "@rollup/rollup-win32-ia32-msvc": ["@rollup/rollup-win32-ia32-msvc@4.45.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-lxV2Pako3ujjuUe9jiU3/s7KSrDfH6IgTSQOnDWr9aJ92YsFd7EurmClK0ly/t8dzMkDtd04g60WX6yl0sGfdw=="], + "@tybys/wasm-util": ["@tybys/wasm-util@0.10.3", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg=="], - "@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.45.1", "", { "os": "win32", "cpu": "x64" }, "sha512-M/fKi4sasCdM8i0aWJjCSFm2qEnYRR8AMLG2kxp6wD13+tMGA4Z1tVAuHkNRjud5SW2EM3naLuK35w9twvf6aA=="], + "@types/bun": ["@types/bun@1.2.18", "", { "dependencies": { "bun-types": "1.2.18" } }, "sha512-Xf6RaWVheyemaThV0kUfaAUvCNokFr+bH8Jxp+tTZfx7dAPA8z9ePnP9S9+Vspzuxxx9JRAXhnyccRj3GyCMdQ=="], - "@sinclair/typebox": ["@sinclair/typebox@0.27.8", "", {}, "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA=="], + "@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="], - "@types/bun": ["@types/bun@1.2.18", "", { "dependencies": { "bun-types": "1.2.18" } }, "sha512-Xf6RaWVheyemaThV0kUfaAUvCNokFr+bH8Jxp+tTZfx7dAPA8z9ePnP9S9+Vspzuxxx9JRAXhnyccRj3GyCMdQ=="], + "@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="], "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], @@ -147,58 +111,52 @@ "@types/react": ["@types/react@19.1.8", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-AwAfQ2Wa5bCx9WP8nZL2uMZWod7J7/JSplxbTmBQ5ms6QpqNYm672H0Vu9ZVKVngQ+ii4R/byguVEUZQyeg44g=="], - "@vitest/expect": ["@vitest/expect@1.6.1", "", { "dependencies": { "@vitest/spy": "1.6.1", "@vitest/utils": "1.6.1", "chai": "^4.3.10" } }, "sha512-jXL+9+ZNIJKruofqXuuTClf44eSpcHlgj3CiuNihUF3Ioujtmc0zIa3UJOW5RjDK1YLBJZnWBlPuqhYycLioog=="], + "@vitest/expect": ["@vitest/expect@4.1.0", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", "@vitest/spy": "4.1.0", "@vitest/utils": "4.1.0", "chai": "^6.2.2", "tinyrainbow": "^3.0.3" } }, "sha512-EIxG7k4wlWweuCLG9Y5InKFwpMEOyrMb6ZJ1ihYu02LVj/bzUwn2VMU+13PinsjRW75XnITeFrQBMH5+dLvCDA=="], - "@vitest/runner": ["@vitest/runner@1.6.1", "", { "dependencies": { "@vitest/utils": "1.6.1", "p-limit": "^5.0.0", "pathe": "^1.1.1" } }, "sha512-3nSnYXkVkf3mXFfE7vVyPmi3Sazhb/2cfZGGs0JRzFsPFvAMBEcrweV1V1GsrstdXeKCTXlJbvnQwGWgEIHmOA=="], + "@vitest/mocker": ["@vitest/mocker@4.1.0", "", { "dependencies": { "@vitest/spy": "4.1.0", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, "peerDependencies": { "msw": "^2.4.9", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0-0" }, "optionalPeers": ["msw", "vite"] }, "sha512-evxREh+Hork43+Y4IOhTo+h5lGmVRyjqI739Rz4RlUPqwrkFFDF6EMvOOYjTx4E8Tl6gyCLRL8Mu7Ry12a13Tw=="], - "@vitest/snapshot": ["@vitest/snapshot@1.6.1", "", { "dependencies": { "magic-string": "^0.30.5", "pathe": "^1.1.1", "pretty-format": "^29.7.0" } }, "sha512-WvidQuWAzU2p95u8GAKlRMqMyN1yOJkGHnx3M1PL9Raf7AQ1kwLKg04ADlCa3+OXUZE7BceOhVZiuWAbzCKcUQ=="], + "@vitest/pretty-format": ["@vitest/pretty-format@4.1.0", "", { "dependencies": { "tinyrainbow": "^3.0.3" } }, "sha512-3RZLZlh88Ib0J7NQTRATfc/3ZPOnSUn2uDBUoGNn5T36+bALixmzphN26OUD3LRXWkJu4H0s5vvUeqBiw+kS0A=="], - "@vitest/spy": ["@vitest/spy@1.6.1", "", { "dependencies": { "tinyspy": "^2.2.0" } }, "sha512-MGcMmpGkZebsMZhbQKkAf9CX5zGvjkBTqf8Zx3ApYWXr3wG+QvEu2eXWfnIIWYSJExIp4V9FCKDEeygzkYrXMw=="], + "@vitest/runner": ["@vitest/runner@4.1.0", "", { "dependencies": { "@vitest/utils": "4.1.0", "pathe": "^2.0.3" } }, "sha512-Duvx2OzQ7d6OjchL+trw+aSrb9idh7pnNfxrklo14p3zmNL4qPCDeIJAK+eBKYjkIwG96Bc6vYuxhqDXQOWpoQ=="], - "@vitest/ui": ["@vitest/ui@1.6.1", "", { "dependencies": { "@vitest/utils": "1.6.1", "fast-glob": "^3.3.2", "fflate": "^0.8.1", "flatted": "^3.2.9", "pathe": "^1.1.1", "picocolors": "^1.0.0", "sirv": "^2.0.4" }, "peerDependencies": { "vitest": "1.6.1" } }, "sha512-xa57bCPGuzEFqGjPs3vVLyqareG8DX0uMkr5U/v5vLv5/ZUrBrPL7gzxzTJedEyZxFMfsozwTIbbYfEQVo3kgg=="], + "@vitest/snapshot": ["@vitest/snapshot@4.1.0", "", { "dependencies": { "@vitest/pretty-format": "4.1.0", "@vitest/utils": "4.1.0", "magic-string": "^0.30.21", "pathe": "^2.0.3" } }, "sha512-0Vy9euT1kgsnj1CHttwi9i9o+4rRLEaPRSOJ5gyv579GJkNpgJK+B4HSv/rAWixx2wdAFci1X4CEPjiu2bXIMg=="], - "@vitest/utils": ["@vitest/utils@1.6.1", "", { "dependencies": { "diff-sequences": "^29.6.3", "estree-walker": "^3.0.3", "loupe": "^2.3.7", "pretty-format": "^29.7.0" } }, "sha512-jOrrUvXM4Av9ZWiG1EajNto0u96kWAhJ1LmPmJhXXQx/32MecEKd10pOLYgS2BQx1TgkGhloPU1ArDW2vvaY6g=="], + "@vitest/spy": ["@vitest/spy@4.1.0", "", {}, "sha512-pz77k+PgNpyMDv2FV6qmk5ZVau6c3R8HC8v342T2xlFxQKTrSeYw9waIJG8KgV9fFwAtTu4ceRzMivPTH6wSxw=="], - "acorn": ["acorn@8.15.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="], + "@vitest/ui": ["@vitest/ui@1.6.1", "", { "dependencies": { "@vitest/utils": "1.6.1", "fast-glob": "^3.3.2", "fflate": "^0.8.1", "flatted": "^3.2.9", "pathe": "^1.1.1", "picocolors": "^1.0.0", "sirv": "^2.0.4" }, "peerDependencies": { "vitest": "1.6.1" } }, "sha512-xa57bCPGuzEFqGjPs3vVLyqareG8DX0uMkr5U/v5vLv5/ZUrBrPL7gzxzTJedEyZxFMfsozwTIbbYfEQVo3kgg=="], - "acorn-walk": ["acorn-walk@8.3.4", "", { "dependencies": { "acorn": "^8.11.0" } }, "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g=="], + "@vitest/utils": ["@vitest/utils@1.6.1", "", { "dependencies": { "diff-sequences": "^29.6.3", "estree-walker": "^3.0.3", "loupe": "^2.3.7", "pretty-format": "^29.7.0" } }, "sha512-jOrrUvXM4Av9ZWiG1EajNto0u96kWAhJ1LmPmJhXXQx/32MecEKd10pOLYgS2BQx1TgkGhloPU1ArDW2vvaY6g=="], "ansi-styles": ["ansi-styles@5.2.0", "", {}, "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA=="], - "assertion-error": ["assertion-error@1.1.0", "", {}, "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw=="], + "assertion-error": ["assertion-error@2.0.1", "", {}, "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA=="], "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], "bun-types": ["bun-types@1.2.18", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-04+Eha5NP7Z0A9YgDAzMk5PHR16ZuLVa83b26kH5+cp1qZW4F6FmAURngE7INf4tKOvCE69vYvDEwoNl1tGiWw=="], - "cac": ["cac@6.7.14", "", {}, "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ=="], - - "chai": ["chai@4.5.0", "", { "dependencies": { "assertion-error": "^1.1.0", "check-error": "^1.0.3", "deep-eql": "^4.1.3", "get-func-name": "^2.0.2", "loupe": "^2.3.6", "pathval": "^1.1.1", "type-detect": "^4.1.0" } }, "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw=="], - - "check-error": ["check-error@1.0.3", "", { "dependencies": { "get-func-name": "^2.0.2" } }, "sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg=="], - - "confbox": ["confbox@0.1.8", "", {}, "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w=="], + "chai": ["chai@6.2.2", "", {}, "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg=="], - "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + "convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="], "csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="], - "debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="], - - "deep-eql": ["deep-eql@4.1.4", "", { "dependencies": { "type-detect": "^4.0.0" } }, "sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg=="], + "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], "diff-sequences": ["diff-sequences@29.6.3", "", {}, "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q=="], - "esbuild": ["esbuild@0.21.5", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.21.5", "@esbuild/android-arm": "0.21.5", "@esbuild/android-arm64": "0.21.5", "@esbuild/android-x64": "0.21.5", "@esbuild/darwin-arm64": "0.21.5", "@esbuild/darwin-x64": "0.21.5", "@esbuild/freebsd-arm64": "0.21.5", "@esbuild/freebsd-x64": "0.21.5", "@esbuild/linux-arm": "0.21.5", "@esbuild/linux-arm64": "0.21.5", "@esbuild/linux-ia32": "0.21.5", "@esbuild/linux-loong64": "0.21.5", "@esbuild/linux-mips64el": "0.21.5", "@esbuild/linux-ppc64": "0.21.5", "@esbuild/linux-riscv64": "0.21.5", "@esbuild/linux-s390x": "0.21.5", "@esbuild/linux-x64": "0.21.5", "@esbuild/netbsd-x64": "0.21.5", "@esbuild/openbsd-x64": "0.21.5", "@esbuild/sunos-x64": "0.21.5", "@esbuild/win32-arm64": "0.21.5", "@esbuild/win32-ia32": "0.21.5", "@esbuild/win32-x64": "0.21.5" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw=="], + "es-module-lexer": ["es-module-lexer@2.3.1", "", {}, "sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA=="], "estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], - "execa": ["execa@8.0.1", "", { "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^8.0.1", "human-signals": "^5.0.0", "is-stream": "^3.0.0", "merge-stream": "^2.0.0", "npm-run-path": "^5.1.0", "onetime": "^6.0.0", "signal-exit": "^4.1.0", "strip-final-newline": "^3.0.0" } }, "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg=="], + "expect-type": ["expect-type@1.4.0", "", {}, "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA=="], "fast-glob": ["fast-glob@3.3.3", "", { "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.8" } }, "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg=="], "fastq": ["fastq@1.19.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ=="], + "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], + "fflate": ["fflate@0.8.2", "", {}, "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A=="], "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], @@ -209,13 +167,9 @@ "get-func-name": ["get-func-name@2.0.2", "", {}, "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ=="], - "get-stream": ["get-stream@8.0.1", "", {}, "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA=="], - "glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], - "hono": ["hono@4.12.7", "", {}, "sha512-jq9l1DM0zVIvsm3lv9Nw9nlJnMNPOcAtsbsgiUhWcFzPE99Gvo6yRTlszSLLYacMeQ6quHD6hMfId8crVHvexw=="], - - "human-signals": ["human-signals@5.0.0", "", {}, "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ=="], + "hono": ["hono@4.12.34", "", {}, "sha512-GqXJqY/xJkJmuloTrnV1ZEXG3fqte+VjkUqoRNZXcrUidiUOP4fMSIHHY4tsqZBK++kVyWmt/AAfSUuy57/eSA=="], "is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="], @@ -223,57 +177,55 @@ "is-number": ["is-number@7.0.0", "", {}, "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="], - "is-stream": ["is-stream@3.0.0", "", {}, "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA=="], + "lightningcss": ["lightningcss@1.33.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.33.0", "lightningcss-darwin-arm64": "1.33.0", "lightningcss-darwin-x64": "1.33.0", "lightningcss-freebsd-x64": "1.33.0", "lightningcss-linux-arm-gnueabihf": "1.33.0", "lightningcss-linux-arm64-gnu": "1.33.0", "lightningcss-linux-arm64-musl": "1.33.0", "lightningcss-linux-x64-gnu": "1.33.0", "lightningcss-linux-x64-musl": "1.33.0", "lightningcss-win32-arm64-msvc": "1.33.0", "lightningcss-win32-x64-msvc": "1.33.0" } }, "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA=="], - "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], + "lightningcss-android-arm64": ["lightningcss-android-arm64@1.33.0", "", { "os": "android", "cpu": "arm64" }, "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg=="], - "js-tokens": ["js-tokens@9.0.1", "", {}, "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ=="], + "lightningcss-darwin-arm64": ["lightningcss-darwin-arm64@1.33.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg=="], - "local-pkg": ["local-pkg@0.5.1", "", { "dependencies": { "mlly": "^1.7.3", "pkg-types": "^1.2.1" } }, "sha512-9rrA30MRRP3gBD3HTGnC6cDFpaE1kVDWxWgqWJUN0RvDNAo+Nz/9GxB+nHOH0ifbVFy0hSA1V6vFDvnx54lTEQ=="], + "lightningcss-darwin-x64": ["lightningcss-darwin-x64@1.33.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ=="], - "loupe": ["loupe@2.3.7", "", { "dependencies": { "get-func-name": "^2.0.1" } }, "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA=="], + "lightningcss-freebsd-x64": ["lightningcss-freebsd-x64@1.33.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg=="], - "magic-string": ["magic-string@0.30.17", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" } }, "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA=="], + "lightningcss-linux-arm-gnueabihf": ["lightningcss-linux-arm-gnueabihf@1.33.0", "", { "os": "linux", "cpu": "arm" }, "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ=="], - "merge-stream": ["merge-stream@2.0.0", "", {}, "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="], + "lightningcss-linux-arm64-gnu": ["lightningcss-linux-arm64-gnu@1.33.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg=="], - "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], + "lightningcss-linux-arm64-musl": ["lightningcss-linux-arm64-musl@1.33.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ=="], - "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], + "lightningcss-linux-x64-gnu": ["lightningcss-linux-x64-gnu@1.33.0", "", { "os": "linux", "cpu": "x64" }, "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg=="], - "mimic-fn": ["mimic-fn@4.0.0", "", {}, "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw=="], + "lightningcss-linux-x64-musl": ["lightningcss-linux-x64-musl@1.33.0", "", { "os": "linux", "cpu": "x64" }, "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw=="], - "mlly": ["mlly@1.7.4", "", { "dependencies": { "acorn": "^8.14.0", "pathe": "^2.0.1", "pkg-types": "^1.3.0", "ufo": "^1.5.4" } }, "sha512-qmdSIPC4bDJXgZTCR7XosJiNKySV7O215tsPtDN9iEO/7q/76b/ijtgRu/+epFXSJhijtTCCGp3DWS549P3xKw=="], + "lightningcss-win32-arm64-msvc": ["lightningcss-win32-arm64-msvc@1.33.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA=="], - "mrmime": ["mrmime@2.0.1", "", {}, "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ=="], + "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.33.0", "", { "os": "win32", "cpu": "x64" }, "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA=="], - "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "loupe": ["loupe@2.3.7", "", { "dependencies": { "get-func-name": "^2.0.1" } }, "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA=="], - "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], + "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], - "npm-run-path": ["npm-run-path@5.3.0", "", { "dependencies": { "path-key": "^4.0.0" } }, "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ=="], + "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], - "onetime": ["onetime@6.0.0", "", { "dependencies": { "mimic-fn": "^4.0.0" } }, "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ=="], + "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], - "openapi-types": ["openapi-types@12.1.3", "", {}, "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw=="], + "mrmime": ["mrmime@2.0.1", "", {}, "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ=="], - "oxlint": ["oxlint@0.9.10", "", { "optionalDependencies": { "@oxlint/darwin-arm64": "0.9.10", "@oxlint/darwin-x64": "0.9.10", "@oxlint/linux-arm64-gnu": "0.9.10", "@oxlint/linux-arm64-musl": "0.9.10", "@oxlint/linux-x64-gnu": "0.9.10", "@oxlint/linux-x64-musl": "0.9.10", "@oxlint/win32-arm64": "0.9.10", "@oxlint/win32-x64": "0.9.10" }, "bin": { "oxlint": "bin/oxlint", "oxc_language_server": "bin/oxc_language_server" } }, "sha512-bKiiFN7Hnoaist/rditTRBXz+GXKYuLd53/NB7Q6zHB/bifELJarSoRLkAUGElIJKl4PSr3lTh1g6zehh+rX0g=="], + "nanoid": ["nanoid@3.3.16", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q=="], - "p-limit": ["p-limit@5.0.0", "", { "dependencies": { "yocto-queue": "^1.0.0" } }, "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ=="], + "obug": ["obug@2.1.4", "", {}, "sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA=="], - "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], + "openapi-types": ["openapi-types@12.1.3", "", {}, "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw=="], - "pathe": ["pathe@1.1.2", "", {}, "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ=="], + "oxlint": ["oxlint@0.9.10", "", { "optionalDependencies": { "@oxlint/darwin-arm64": "0.9.10", "@oxlint/darwin-x64": "0.9.10", "@oxlint/linux-arm64-gnu": "0.9.10", "@oxlint/linux-arm64-musl": "0.9.10", "@oxlint/linux-x64-gnu": "0.9.10", "@oxlint/linux-x64-musl": "0.9.10", "@oxlint/win32-arm64": "0.9.10", "@oxlint/win32-x64": "0.9.10" }, "bin": { "oxlint": "bin/oxlint", "oxc_language_server": "bin/oxc_language_server" } }, "sha512-bKiiFN7Hnoaist/rditTRBXz+GXKYuLd53/NB7Q6zHB/bifELJarSoRLkAUGElIJKl4PSr3lTh1g6zehh+rX0g=="], - "pathval": ["pathval@1.1.1", "", {}, "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ=="], + "pathe": ["pathe@1.1.2", "", {}, "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ=="], "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], - "picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + "picomatch": ["picomatch@4.0.5", "", {}, "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A=="], - "pkg-types": ["pkg-types@1.3.1", "", { "dependencies": { "confbox": "^0.1.8", "mlly": "^1.7.4", "pathe": "^2.0.1" } }, "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ=="], - - "postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="], + "postcss": ["postcss@8.5.23", "", { "dependencies": { "nanoid": "^3.3.16", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg=="], "pretty-format": ["pretty-format@29.7.0", "", { "dependencies": { "@jest/schemas": "^29.6.3", "ansi-styles": "^5.0.0", "react-is": "^18.0.0" } }, "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ=="], @@ -283,69 +235,63 @@ "reusify": ["reusify@1.1.0", "", {}, "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw=="], - "rollup": ["rollup@4.45.1", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.45.1", "@rollup/rollup-android-arm64": "4.45.1", "@rollup/rollup-darwin-arm64": "4.45.1", "@rollup/rollup-darwin-x64": "4.45.1", "@rollup/rollup-freebsd-arm64": "4.45.1", "@rollup/rollup-freebsd-x64": "4.45.1", "@rollup/rollup-linux-arm-gnueabihf": "4.45.1", "@rollup/rollup-linux-arm-musleabihf": "4.45.1", "@rollup/rollup-linux-arm64-gnu": "4.45.1", "@rollup/rollup-linux-arm64-musl": "4.45.1", "@rollup/rollup-linux-loongarch64-gnu": "4.45.1", "@rollup/rollup-linux-powerpc64le-gnu": "4.45.1", "@rollup/rollup-linux-riscv64-gnu": "4.45.1", "@rollup/rollup-linux-riscv64-musl": "4.45.1", "@rollup/rollup-linux-s390x-gnu": "4.45.1", "@rollup/rollup-linux-x64-gnu": "4.45.1", "@rollup/rollup-linux-x64-musl": "4.45.1", "@rollup/rollup-win32-arm64-msvc": "4.45.1", "@rollup/rollup-win32-ia32-msvc": "4.45.1", "@rollup/rollup-win32-x64-msvc": "4.45.1", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-4iya7Jb76fVpQyLoiVpzUrsjQ12r3dM7fIVz+4NwoYvZOShknRmiv+iu9CClZml5ZLGb0XMcYLutK6w9tgxHDw=="], + "rolldown": ["rolldown@1.1.5", "", { "dependencies": { "@oxc-project/types": "=0.139.0", "@rolldown/pluginutils": "^1.0.0" }, "optionalDependencies": { "@rolldown/binding-android-arm64": "1.1.5", "@rolldown/binding-darwin-arm64": "1.1.5", "@rolldown/binding-darwin-x64": "1.1.5", "@rolldown/binding-freebsd-x64": "1.1.5", "@rolldown/binding-linux-arm-gnueabihf": "1.1.5", "@rolldown/binding-linux-arm64-gnu": "1.1.5", "@rolldown/binding-linux-arm64-musl": "1.1.5", "@rolldown/binding-linux-ppc64-gnu": "1.1.5", "@rolldown/binding-linux-s390x-gnu": "1.1.5", "@rolldown/binding-linux-x64-gnu": "1.1.5", "@rolldown/binding-linux-x64-musl": "1.1.5", "@rolldown/binding-openharmony-arm64": "1.1.5", "@rolldown/binding-wasm32-wasi": "1.1.5", "@rolldown/binding-win32-arm64-msvc": "1.1.5", "@rolldown/binding-win32-x64-msvc": "1.1.5" }, "bin": { "rolldown": "./bin/cli.mjs" } }, "sha512-t9z29cJjXf/vxQ8dyhCSpt6H6aSwHTk8cT5I3iy6SMXuFpk5mB6PL6XfC8PCwrPTx93udwKUm9HRteAlTGBLiA=="], "run-parallel": ["run-parallel@1.2.0", "", { "dependencies": { "queue-microtask": "^1.2.2" } }, "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA=="], - "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], - - "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], - "siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="], - "signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], - "sirv": ["sirv@2.0.4", "", { "dependencies": { "@polka/url": "^1.0.0-next.24", "mrmime": "^2.0.0", "totalist": "^3.0.0" } }, "sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ=="], "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], "stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="], - "std-env": ["std-env@3.9.0", "", {}, "sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw=="], - - "strip-final-newline": ["strip-final-newline@3.0.0", "", {}, "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw=="], - - "strip-literal": ["strip-literal@2.1.1", "", { "dependencies": { "js-tokens": "^9.0.1" } }, "sha512-631UJ6O00eNGfMiWG78ck80dfBab8X6IVFB51jZK5Icd7XAs60Z5y7QdSd/wGIklnWvRbUNloVzhOKKmutxQ6Q=="], + "std-env": ["std-env@4.2.0", "", {}, "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw=="], "tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="], - "tinypool": ["tinypool@0.8.4", "", {}, "sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ=="], + "tinyexec": ["tinyexec@1.2.4", "", {}, "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg=="], - "tinyspy": ["tinyspy@2.2.1", "", {}, "sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A=="], + "tinyglobby": ["tinyglobby@0.2.17", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="], + + "tinyrainbow": ["tinyrainbow@3.1.0", "", {}, "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw=="], "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], "totalist": ["totalist@3.0.1", "", {}, "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ=="], - "type-detect": ["type-detect@4.1.0", "", {}, "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw=="], + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], "typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="], - "ufo": ["ufo@1.6.1", "", {}, "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA=="], - "undici-types": ["undici-types@6.20.0", "", {}, "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="], - "vite": ["vite@5.4.19", "", { "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", "rollup": "^4.20.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || >=20.0.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" }, "optionalPeers": ["@types/node", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser"], "bin": { "vite": "bin/vite.js" } }, "sha512-qO3aKv3HoQC8QKiNSTuUM1l9o/XX3+c+VTgLHbJWHZGeTPVAg2XwazI9UWzoxjIJCGCV2zU60uqMzjeLZuULqA=="], + "vite": ["vite@8.1.5", "", { "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.5", "postcss": "^8.5.17", "rolldown": "~1.1.5", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.3.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw=="], + + "vitest": ["vitest@4.1.0", "", { "dependencies": { "@vitest/expect": "4.1.0", "@vitest/mocker": "4.1.0", "@vitest/pretty-format": "4.1.0", "@vitest/runner": "4.1.0", "@vitest/snapshot": "4.1.0", "@vitest/spy": "4.1.0", "@vitest/utils": "4.1.0", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", "obug": "^2.1.1", "pathe": "^2.0.3", "picomatch": "^4.0.3", "std-env": "^4.0.0-rc.1", "tinybench": "^2.9.0", "tinyexec": "^1.0.2", "tinyglobby": "^0.2.15", "tinyrainbow": "^3.0.3", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0-0", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", "@vitest/browser-playwright": "4.1.0", "@vitest/browser-preview": "4.1.0", "@vitest/browser-webdriverio": "4.1.0", "@vitest/ui": "4.1.0", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@opentelemetry/api", "@types/node", "@vitest/browser-playwright", "@vitest/browser-preview", "@vitest/browser-webdriverio", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-YbDrMF9jM2Lqc++2530UourxZHmkKLxrs4+mYhEwqWS97WJ7wOYEkcr+QfRgJ3PW9wz3odRijLZjHEaRLTNbqw=="], + + "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], - "vite-node": ["vite-node@1.6.1", "", { "dependencies": { "cac": "^6.7.14", "debug": "^4.3.4", "pathe": "^1.1.1", "picocolors": "^1.0.0", "vite": "^5.0.0" }, "bin": { "vite-node": "vite-node.mjs" } }, "sha512-YAXkfvGtuTzwWbDSACdJSg4A4DZiAqckWe90Zapc/sEX3XvHcw1NdurM/6od8J207tSDqNbSsgdCacBgvJKFuA=="], + "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], - "vitest": ["vitest@1.6.1", "", { "dependencies": { "@vitest/expect": "1.6.1", "@vitest/runner": "1.6.1", "@vitest/snapshot": "1.6.1", "@vitest/spy": "1.6.1", "@vitest/utils": "1.6.1", "acorn-walk": "^8.3.2", "chai": "^4.3.10", "debug": "^4.3.4", "execa": "^8.0.1", "local-pkg": "^0.5.0", "magic-string": "^0.30.5", "pathe": "^1.1.1", "picocolors": "^1.0.0", "std-env": "^3.5.0", "strip-literal": "^2.0.0", "tinybench": "^2.5.1", "tinypool": "^0.8.3", "vite": "^5.0.0", "vite-node": "1.6.1", "why-is-node-running": "^2.2.2" }, "peerDependencies": { "@edge-runtime/vm": "*", "@types/node": "^18.0.0 || >=20.0.0", "@vitest/browser": "1.6.1", "@vitest/ui": "1.6.1", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@types/node", "@vitest/browser", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-Ljb1cnSJSivGN0LqXd/zmDbWEM0RNNg2t1QW/XUhYl/qPqyu7CsqeWtqQXHVaJsecLPuDoak2oJcZN2QoRIOag=="], + "@vitest/expect/@vitest/utils": ["@vitest/utils@4.1.0", "", { "dependencies": { "@vitest/pretty-format": "4.1.0", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.0.3" } }, "sha512-XfPXT6a8TZY3dcGY8EdwsBulFCIw+BeeX0RZn2x/BtiY/75YGh8FeWGG8QISN/WhaqSrE2OrlDgtF8q5uhOTmw=="], - "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], + "@vitest/runner/@vitest/utils": ["@vitest/utils@4.1.0", "", { "dependencies": { "@vitest/pretty-format": "4.1.0", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.0.3" } }, "sha512-XfPXT6a8TZY3dcGY8EdwsBulFCIw+BeeX0RZn2x/BtiY/75YGh8FeWGG8QISN/WhaqSrE2OrlDgtF8q5uhOTmw=="], - "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], + "@vitest/runner/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], - "yocto-queue": ["yocto-queue@1.2.1", "", {}, "sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg=="], + "@vitest/snapshot/@vitest/utils": ["@vitest/utils@4.1.0", "", { "dependencies": { "@vitest/pretty-format": "4.1.0", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.0.3" } }, "sha512-XfPXT6a8TZY3dcGY8EdwsBulFCIw+BeeX0RZn2x/BtiY/75YGh8FeWGG8QISN/WhaqSrE2OrlDgtF8q5uhOTmw=="], - "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + "@vitest/snapshot/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], "bun-types/@types/node": ["@types/node@24.0.14", "", { "dependencies": { "undici-types": "~7.8.0" } }, "sha512-4zXMWD91vBLGRtHK3YbIoFMia+1nqEz72coM42C5ETjnNCa/heoj7NT1G67iAfOqMmcfhuCZ4uNpyz8EjlAejw=="], - "mlly/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], - "npm-run-path/path-key": ["path-key@4.0.0", "", {}, "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ=="], + "vitest/@vitest/utils": ["@vitest/utils@4.1.0", "", { "dependencies": { "@vitest/pretty-format": "4.1.0", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.0.3" } }, "sha512-XfPXT6a8TZY3dcGY8EdwsBulFCIw+BeeX0RZn2x/BtiY/75YGh8FeWGG8QISN/WhaqSrE2OrlDgtF8q5uhOTmw=="], - "pkg-types/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + "vitest/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], "bun-types/@types/node/undici-types": ["undici-types@7.8.0", "", {}, "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw=="], } diff --git a/dstack/kms/auth-mock/index.ts b/dstack/kms/auth-mock/index.ts index a2c5b3854..f53d4bbd5 100644 --- a/dstack/kms/auth-mock/index.ts +++ b/dstack/kms/auth-mock/index.ts @@ -6,6 +6,10 @@ import { Hono } from 'hono'; import { zValidator } from '@hono/zod-validator'; import { z } from 'zod'; +if (process.env.NODE_ENV === 'production') { + throw new Error('auth-mock must not run with NODE_ENV=production'); +} + // zod schemas for validation - compatible with original fastify implementation const BootInfoSchema = z.object({ // required fields (matching original fastify schema) diff --git a/dstack/kms/auth-mock/package.json b/dstack/kms/auth-mock/package.json index 7a175cef6..09094da99 100644 --- a/dstack/kms/auth-mock/package.json +++ b/dstack/kms/auth-mock/package.json @@ -15,7 +15,7 @@ "check": "bun run lint && bun run test:run" }, "dependencies": { - "hono": "4.12.25", + "hono": "4.13.5", "@hono/zod-validator": "0.2.2", "zod": "3.25.76" }, @@ -26,7 +26,7 @@ "openapi-types": "12.1.3", "oxlint": "0.9.10", "typescript": "5.8.3", - "vitest": "4.1.0" + "vitest": "4.1.11" }, "type": "module" } diff --git a/dstack/kms/auth-simple/README.md b/dstack/kms/auth-simple/README.md index f7c826e8f..6857122d1 100644 --- a/dstack/kms/auth-simple/README.md +++ b/dstack/kms/auth-simple/README.md @@ -171,7 +171,7 @@ App boot authorization. **Request:** ```json { - "attestationMode": "dstack-tdx", + "teeVariant": "dstack-tdx", "mrAggregated": "0x...", "osImageHash": "0x...", "appId": "0x...", diff --git a/dstack/kms/auth-simple/bun.lock b/dstack/kms/auth-simple/bun.lock index eb4a4740d..dc65b4382 100644 --- a/dstack/kms/auth-simple/bun.lock +++ b/dstack/kms/auth-simple/bun.lock @@ -1,12 +1,11 @@ { "lockfileVersion": 1, - "configVersion": 1, "workspaces": { "": { "name": "auth-simple", "dependencies": { "@hono/zod-validator": "0.2.2", - "hono": "4.12.7", + "hono": "4.12.34", "zod": "3.25.76", }, "devDependencies": { @@ -16,7 +15,7 @@ "openapi-types": "12.1.3", "oxlint": "0.9.10", "typescript": "5.8.3", - "vitest": "1.6.1", + "vitest": "3.2.6", }, }, }, @@ -145,31 +144,35 @@ "@types/bun": ["@types/bun@1.2.18", "", { "dependencies": { "bun-types": "1.2.18" } }, "sha512-Xf6RaWVheyemaThV0kUfaAUvCNokFr+bH8Jxp+tTZfx7dAPA8z9ePnP9S9+Vspzuxxx9JRAXhnyccRj3GyCMdQ=="], + "@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="], + + "@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="], + "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], "@types/node": ["@types/node@22.10.8", "", { "dependencies": { "undici-types": "~6.20.0" } }, "sha512-rk+QvAEGsbX/ZPiiyel6hJHNUS9cnSbPWVaZLvE+Er3tLqQFzWMz9JOfWW7XUmKvRPfxJfbl3qYWve+RGXncFw=="], "@types/react": ["@types/react@19.2.7", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-MWtvHrGZLFttgeEj28VXHxpmwYbor/ATPYbBfSFZEIRK0ecCFLl2Qo55z52Hss+UV9CRN7trSeq1zbgx7YDWWg=="], - "@vitest/expect": ["@vitest/expect@1.6.1", "", { "dependencies": { "@vitest/spy": "1.6.1", "@vitest/utils": "1.6.1", "chai": "^4.3.10" } }, "sha512-jXL+9+ZNIJKruofqXuuTClf44eSpcHlgj3CiuNihUF3Ioujtmc0zIa3UJOW5RjDK1YLBJZnWBlPuqhYycLioog=="], + "@vitest/expect": ["@vitest/expect@3.2.6", "", { "dependencies": { "@types/chai": "^5.2.2", "@vitest/spy": "3.2.6", "@vitest/utils": "3.2.6", "chai": "^5.2.0", "tinyrainbow": "^2.0.0" } }, "sha512-1+7q9BtaKzEmO+fmNT3kYvoNn5Y71XWAx2Q5HRim4tTVRQVRv4uJFAQ5FbK0OPUeNP/WmVCpxYxoJdvuHVjzBQ=="], - "@vitest/runner": ["@vitest/runner@1.6.1", "", { "dependencies": { "@vitest/utils": "1.6.1", "p-limit": "^5.0.0", "pathe": "^1.1.1" } }, "sha512-3nSnYXkVkf3mXFfE7vVyPmi3Sazhb/2cfZGGs0JRzFsPFvAMBEcrweV1V1GsrstdXeKCTXlJbvnQwGWgEIHmOA=="], + "@vitest/mocker": ["@vitest/mocker@3.2.6", "", { "dependencies": { "@vitest/spy": "3.2.6", "estree-walker": "^3.0.3", "magic-string": "^0.30.17" }, "peerDependencies": { "msw": "^2.4.9", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" }, "optionalPeers": ["msw", "vite"] }, "sha512-EZOrpDbkKotFAP7wPAQV1UIyoGOk4oX7ynWhBhLB7v+meMHbQhU16oPpIYGTTe4oFlhpryGpgpcZP/sin3hYuw=="], - "@vitest/snapshot": ["@vitest/snapshot@1.6.1", "", { "dependencies": { "magic-string": "^0.30.5", "pathe": "^1.1.1", "pretty-format": "^29.7.0" } }, "sha512-WvidQuWAzU2p95u8GAKlRMqMyN1yOJkGHnx3M1PL9Raf7AQ1kwLKg04ADlCa3+OXUZE7BceOhVZiuWAbzCKcUQ=="], + "@vitest/pretty-format": ["@vitest/pretty-format@3.2.7", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA=="], - "@vitest/spy": ["@vitest/spy@1.6.1", "", { "dependencies": { "tinyspy": "^2.2.0" } }, "sha512-MGcMmpGkZebsMZhbQKkAf9CX5zGvjkBTqf8Zx3ApYWXr3wG+QvEu2eXWfnIIWYSJExIp4V9FCKDEeygzkYrXMw=="], + "@vitest/runner": ["@vitest/runner@3.2.6", "", { "dependencies": { "@vitest/utils": "3.2.6", "pathe": "^2.0.3", "strip-literal": "^3.0.0" } }, "sha512-HYcoSj1w5tcgUnzoF0HcyaAQjpA1gj9ftUJ7iSJSuipc02jW9gKkigwZbjFldAfYHA1fa8UZVRftdMY5msWM9Q=="], - "@vitest/ui": ["@vitest/ui@1.6.1", "", { "dependencies": { "@vitest/utils": "1.6.1", "fast-glob": "^3.3.2", "fflate": "^0.8.1", "flatted": "^3.2.9", "pathe": "^1.1.1", "picocolors": "^1.0.0", "sirv": "^2.0.4" }, "peerDependencies": { "vitest": "1.6.1" } }, "sha512-xa57bCPGuzEFqGjPs3vVLyqareG8DX0uMkr5U/v5vLv5/ZUrBrPL7gzxzTJedEyZxFMfsozwTIbbYfEQVo3kgg=="], + "@vitest/snapshot": ["@vitest/snapshot@3.2.6", "", { "dependencies": { "@vitest/pretty-format": "3.2.6", "magic-string": "^0.30.17", "pathe": "^2.0.3" } }, "sha512-H+ZjNTWGpObenh0YnlBctAPnJSI20P81PL8BPzWpx54YXLLTm8hEsWawtcYLMrwvpK48hGxLLbCS+1KRXhsKhw=="], - "@vitest/utils": ["@vitest/utils@1.6.1", "", { "dependencies": { "diff-sequences": "^29.6.3", "estree-walker": "^3.0.3", "loupe": "^2.3.7", "pretty-format": "^29.7.0" } }, "sha512-jOrrUvXM4Av9ZWiG1EajNto0u96kWAhJ1LmPmJhXXQx/32MecEKd10pOLYgS2BQx1TgkGhloPU1ArDW2vvaY6g=="], + "@vitest/spy": ["@vitest/spy@3.2.6", "", { "dependencies": { "tinyspy": "^4.0.3" } }, "sha512-oq6BbH68WzcWmwtBrU9nqLeaXTR4XwJF7FSLkKEZo4i6eoXcrxjcwSuTvWBIRUTC6VC72nXYunzqgZA+IKdtxg=="], - "acorn": ["acorn@8.15.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="], + "@vitest/ui": ["@vitest/ui@1.6.1", "", { "dependencies": { "@vitest/utils": "1.6.1", "fast-glob": "^3.3.2", "fflate": "^0.8.1", "flatted": "^3.2.9", "pathe": "^1.1.1", "picocolors": "^1.0.0", "sirv": "^2.0.4" }, "peerDependencies": { "vitest": "1.6.1" } }, "sha512-xa57bCPGuzEFqGjPs3vVLyqareG8DX0uMkr5U/v5vLv5/ZUrBrPL7gzxzTJedEyZxFMfsozwTIbbYfEQVo3kgg=="], - "acorn-walk": ["acorn-walk@8.3.4", "", { "dependencies": { "acorn": "^8.11.0" } }, "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g=="], + "@vitest/utils": ["@vitest/utils@1.6.1", "", { "dependencies": { "diff-sequences": "^29.6.3", "estree-walker": "^3.0.3", "loupe": "^2.3.7", "pretty-format": "^29.7.0" } }, "sha512-jOrrUvXM4Av9ZWiG1EajNto0u96kWAhJ1LmPmJhXXQx/32MecEKd10pOLYgS2BQx1TgkGhloPU1ArDW2vvaY6g=="], "ansi-styles": ["ansi-styles@5.2.0", "", {}, "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA=="], - "assertion-error": ["assertion-error@1.1.0", "", {}, "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw=="], + "assertion-error": ["assertion-error@2.0.1", "", {}, "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA=="], "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], @@ -177,32 +180,32 @@ "cac": ["cac@6.7.14", "", {}, "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ=="], - "chai": ["chai@4.5.0", "", { "dependencies": { "assertion-error": "^1.1.0", "check-error": "^1.0.3", "deep-eql": "^4.1.3", "get-func-name": "^2.0.2", "loupe": "^2.3.6", "pathval": "^1.1.1", "type-detect": "^4.1.0" } }, "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw=="], + "chai": ["chai@5.3.3", "", { "dependencies": { "assertion-error": "^2.0.1", "check-error": "^2.1.1", "deep-eql": "^5.0.1", "loupe": "^3.1.0", "pathval": "^2.0.0" } }, "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw=="], - "check-error": ["check-error@1.0.3", "", { "dependencies": { "get-func-name": "^2.0.2" } }, "sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg=="], - - "confbox": ["confbox@0.1.8", "", {}, "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w=="], - - "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + "check-error": ["check-error@2.1.3", "", {}, "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA=="], "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], - "deep-eql": ["deep-eql@4.1.4", "", { "dependencies": { "type-detect": "^4.0.0" } }, "sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg=="], + "deep-eql": ["deep-eql@5.0.2", "", {}, "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q=="], "diff-sequences": ["diff-sequences@29.6.3", "", {}, "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q=="], + "es-module-lexer": ["es-module-lexer@1.7.0", "", {}, "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA=="], + "esbuild": ["esbuild@0.21.5", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.21.5", "@esbuild/android-arm": "0.21.5", "@esbuild/android-arm64": "0.21.5", "@esbuild/android-x64": "0.21.5", "@esbuild/darwin-arm64": "0.21.5", "@esbuild/darwin-x64": "0.21.5", "@esbuild/freebsd-arm64": "0.21.5", "@esbuild/freebsd-x64": "0.21.5", "@esbuild/linux-arm": "0.21.5", "@esbuild/linux-arm64": "0.21.5", "@esbuild/linux-ia32": "0.21.5", "@esbuild/linux-loong64": "0.21.5", "@esbuild/linux-mips64el": "0.21.5", "@esbuild/linux-ppc64": "0.21.5", "@esbuild/linux-riscv64": "0.21.5", "@esbuild/linux-s390x": "0.21.5", "@esbuild/linux-x64": "0.21.5", "@esbuild/netbsd-x64": "0.21.5", "@esbuild/openbsd-x64": "0.21.5", "@esbuild/sunos-x64": "0.21.5", "@esbuild/win32-arm64": "0.21.5", "@esbuild/win32-ia32": "0.21.5", "@esbuild/win32-x64": "0.21.5" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw=="], "estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], - "execa": ["execa@8.0.1", "", { "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^8.0.1", "human-signals": "^5.0.0", "is-stream": "^3.0.0", "merge-stream": "^2.0.0", "npm-run-path": "^5.1.0", "onetime": "^6.0.0", "signal-exit": "^4.1.0", "strip-final-newline": "^3.0.0" } }, "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg=="], + "expect-type": ["expect-type@1.4.0", "", {}, "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA=="], "fast-glob": ["fast-glob@3.3.3", "", { "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.8" } }, "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg=="], "fastq": ["fastq@1.20.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw=="], + "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], + "fflate": ["fflate@0.8.2", "", {}, "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A=="], "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], @@ -213,13 +216,9 @@ "get-func-name": ["get-func-name@2.0.2", "", {}, "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ=="], - "get-stream": ["get-stream@8.0.1", "", {}, "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA=="], - "glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], - "hono": ["hono@4.12.7", "", {}, "sha512-jq9l1DM0zVIvsm3lv9Nw9nlJnMNPOcAtsbsgiUhWcFzPE99Gvo6yRTlszSLLYacMeQ6quHD6hMfId8crVHvexw=="], - - "human-signals": ["human-signals@5.0.0", "", {}, "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ=="], + "hono": ["hono@4.12.34", "", {}, "sha512-GqXJqY/xJkJmuloTrnV1ZEXG3fqte+VjkUqoRNZXcrUidiUOP4fMSIHHY4tsqZBK++kVyWmt/AAfSUuy57/eSA=="], "is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="], @@ -227,55 +226,33 @@ "is-number": ["is-number@7.0.0", "", {}, "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="], - "is-stream": ["is-stream@3.0.0", "", {}, "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA=="], - - "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], - "js-tokens": ["js-tokens@9.0.1", "", {}, "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ=="], - "local-pkg": ["local-pkg@0.5.1", "", { "dependencies": { "mlly": "^1.7.3", "pkg-types": "^1.2.1" } }, "sha512-9rrA30MRRP3gBD3HTGnC6cDFpaE1kVDWxWgqWJUN0RvDNAo+Nz/9GxB+nHOH0ifbVFy0hSA1V6vFDvnx54lTEQ=="], - "loupe": ["loupe@2.3.7", "", { "dependencies": { "get-func-name": "^2.0.1" } }, "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA=="], "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], - "merge-stream": ["merge-stream@2.0.0", "", {}, "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="], - "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], - "mimic-fn": ["mimic-fn@4.0.0", "", {}, "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw=="], - - "mlly": ["mlly@1.8.0", "", { "dependencies": { "acorn": "^8.15.0", "pathe": "^2.0.3", "pkg-types": "^1.3.1", "ufo": "^1.6.1" } }, "sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g=="], - "mrmime": ["mrmime@2.0.1", "", {}, "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ=="], "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], - "npm-run-path": ["npm-run-path@5.3.0", "", { "dependencies": { "path-key": "^4.0.0" } }, "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ=="], - - "onetime": ["onetime@6.0.0", "", { "dependencies": { "mimic-fn": "^4.0.0" } }, "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ=="], - "openapi-types": ["openapi-types@12.1.3", "", {}, "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw=="], "oxlint": ["oxlint@0.9.10", "", { "optionalDependencies": { "@oxlint/darwin-arm64": "0.9.10", "@oxlint/darwin-x64": "0.9.10", "@oxlint/linux-arm64-gnu": "0.9.10", "@oxlint/linux-arm64-musl": "0.9.10", "@oxlint/linux-x64-gnu": "0.9.10", "@oxlint/linux-x64-musl": "0.9.10", "@oxlint/win32-arm64": "0.9.10", "@oxlint/win32-x64": "0.9.10" }, "bin": { "oxlint": "bin/oxlint", "oxc_language_server": "bin/oxc_language_server" } }, "sha512-bKiiFN7Hnoaist/rditTRBXz+GXKYuLd53/NB7Q6zHB/bifELJarSoRLkAUGElIJKl4PSr3lTh1g6zehh+rX0g=="], - "p-limit": ["p-limit@5.0.0", "", { "dependencies": { "yocto-queue": "^1.0.0" } }, "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ=="], - - "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], - "pathe": ["pathe@1.1.2", "", {}, "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ=="], - "pathval": ["pathval@1.1.1", "", {}, "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ=="], + "pathval": ["pathval@2.0.1", "", {}, "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ=="], "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], - "picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], - - "pkg-types": ["pkg-types@1.3.1", "", { "dependencies": { "confbox": "^0.1.8", "mlly": "^1.7.4", "pathe": "^2.0.1" } }, "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ=="], + "picomatch": ["picomatch@4.0.5", "", {}, "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A=="], "postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="], @@ -291,14 +268,8 @@ "run-parallel": ["run-parallel@1.2.0", "", { "dependencies": { "queue-microtask": "^1.2.2" } }, "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA=="], - "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], - - "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], - "siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="], - "signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], - "sirv": ["sirv@2.0.4", "", { "dependencies": { "@polka/url": "^1.0.0-next.24", "mrmime": "^2.0.0", "totalist": "^3.0.0" } }, "sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ=="], "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], @@ -307,46 +278,68 @@ "std-env": ["std-env@3.10.0", "", {}, "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg=="], - "strip-final-newline": ["strip-final-newline@3.0.0", "", {}, "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw=="], - - "strip-literal": ["strip-literal@2.1.1", "", { "dependencies": { "js-tokens": "^9.0.1" } }, "sha512-631UJ6O00eNGfMiWG78ck80dfBab8X6IVFB51jZK5Icd7XAs60Z5y7QdSd/wGIklnWvRbUNloVzhOKKmutxQ6Q=="], + "strip-literal": ["strip-literal@3.1.0", "", { "dependencies": { "js-tokens": "^9.0.1" } }, "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg=="], "tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="], - "tinypool": ["tinypool@0.8.4", "", {}, "sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ=="], + "tinyexec": ["tinyexec@0.3.2", "", {}, "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA=="], + + "tinyglobby": ["tinyglobby@0.2.17", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="], + + "tinypool": ["tinypool@1.1.1", "", {}, "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg=="], + + "tinyrainbow": ["tinyrainbow@2.0.0", "", {}, "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw=="], - "tinyspy": ["tinyspy@2.2.1", "", {}, "sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A=="], + "tinyspy": ["tinyspy@4.0.4", "", {}, "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q=="], "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], "totalist": ["totalist@3.0.1", "", {}, "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ=="], - "type-detect": ["type-detect@4.1.0", "", {}, "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw=="], - "typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="], - "ufo": ["ufo@1.6.1", "", {}, "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA=="], - "undici-types": ["undici-types@6.20.0", "", {}, "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="], "vite": ["vite@5.4.21", "", { "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", "rollup": "^4.20.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || >=20.0.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" }, "optionalPeers": ["@types/node", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser"], "bin": { "vite": "bin/vite.js" } }, "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw=="], - "vite-node": ["vite-node@1.6.1", "", { "dependencies": { "cac": "^6.7.14", "debug": "^4.3.4", "pathe": "^1.1.1", "picocolors": "^1.0.0", "vite": "^5.0.0" }, "bin": { "vite-node": "vite-node.mjs" } }, "sha512-YAXkfvGtuTzwWbDSACdJSg4A4DZiAqckWe90Zapc/sEX3XvHcw1NdurM/6od8J207tSDqNbSsgdCacBgvJKFuA=="], - - "vitest": ["vitest@1.6.1", "", { "dependencies": { "@vitest/expect": "1.6.1", "@vitest/runner": "1.6.1", "@vitest/snapshot": "1.6.1", "@vitest/spy": "1.6.1", "@vitest/utils": "1.6.1", "acorn-walk": "^8.3.2", "chai": "^4.3.10", "debug": "^4.3.4", "execa": "^8.0.1", "local-pkg": "^0.5.0", "magic-string": "^0.30.5", "pathe": "^1.1.1", "picocolors": "^1.0.0", "std-env": "^3.5.0", "strip-literal": "^2.0.0", "tinybench": "^2.5.1", "tinypool": "^0.8.3", "vite": "^5.0.0", "vite-node": "1.6.1", "why-is-node-running": "^2.2.2" }, "peerDependencies": { "@edge-runtime/vm": "*", "@types/node": "^18.0.0 || >=20.0.0", "@vitest/browser": "1.6.1", "@vitest/ui": "1.6.1", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@types/node", "@vitest/browser", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-Ljb1cnSJSivGN0LqXd/zmDbWEM0RNNg2t1QW/XUhYl/qPqyu7CsqeWtqQXHVaJsecLPuDoak2oJcZN2QoRIOag=="], + "vite-node": ["vite-node@3.2.4", "", { "dependencies": { "cac": "^6.7.14", "debug": "^4.4.1", "es-module-lexer": "^1.7.0", "pathe": "^2.0.3", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" }, "bin": { "vite-node": "vite-node.mjs" } }, "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg=="], - "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], + "vitest": ["vitest@3.2.6", "", { "dependencies": { "@types/chai": "^5.2.2", "@vitest/expect": "3.2.6", "@vitest/mocker": "3.2.6", "@vitest/pretty-format": "^3.2.6", "@vitest/runner": "3.2.6", "@vitest/snapshot": "3.2.6", "@vitest/spy": "3.2.6", "@vitest/utils": "3.2.6", "chai": "^5.2.0", "debug": "^4.4.1", "expect-type": "^1.2.1", "magic-string": "^0.30.17", "pathe": "^2.0.3", "picomatch": "^4.0.2", "std-env": "^3.9.0", "tinybench": "^2.9.0", "tinyexec": "^0.3.2", "tinyglobby": "^0.2.14", "tinypool": "^1.1.1", "tinyrainbow": "^2.0.0", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", "vite-node": "3.2.4", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@types/debug": "^4.1.12", "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "@vitest/browser": "3.2.6", "@vitest/ui": "3.2.6", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@types/debug", "@types/node", "@vitest/browser", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-xejya+bT/j/+R/AGa1XOfRxLmNUlLtlwjRsFUILF+xHfzElmGcmFydy2gqqIrd62ptIEfwVMofd19uNWD9L7Nw=="], "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], - "yocto-queue": ["yocto-queue@1.2.2", "", {}, "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ=="], - "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], - "mlly/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + "@vitest/expect/@vitest/utils": ["@vitest/utils@3.2.6", "", { "dependencies": { "@vitest/pretty-format": "3.2.6", "loupe": "^3.1.4", "tinyrainbow": "^2.0.0" } }, "sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg=="], + + "@vitest/runner/@vitest/utils": ["@vitest/utils@3.2.6", "", { "dependencies": { "@vitest/pretty-format": "3.2.6", "loupe": "^3.1.4", "tinyrainbow": "^2.0.0" } }, "sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg=="], + + "@vitest/runner/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + + "@vitest/snapshot/@vitest/pretty-format": ["@vitest/pretty-format@3.2.6", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA=="], + + "@vitest/snapshot/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + + "chai/loupe": ["loupe@3.2.1", "", {}, "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ=="], + + "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + + "vite-node/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + + "vitest/@vitest/utils": ["@vitest/utils@3.2.6", "", { "dependencies": { "@vitest/pretty-format": "3.2.6", "loupe": "^3.1.4", "tinyrainbow": "^2.0.0" } }, "sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg=="], + + "vitest/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + + "@vitest/expect/@vitest/utils/@vitest/pretty-format": ["@vitest/pretty-format@3.2.6", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA=="], + + "@vitest/expect/@vitest/utils/loupe": ["loupe@3.2.1", "", {}, "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ=="], + + "@vitest/runner/@vitest/utils/@vitest/pretty-format": ["@vitest/pretty-format@3.2.6", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA=="], + + "@vitest/runner/@vitest/utils/loupe": ["loupe@3.2.1", "", {}, "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ=="], - "npm-run-path/path-key": ["path-key@4.0.0", "", {}, "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ=="], + "vitest/@vitest/utils/@vitest/pretty-format": ["@vitest/pretty-format@3.2.6", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA=="], - "pkg-types/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + "vitest/@vitest/utils/loupe": ["loupe@3.2.1", "", {}, "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ=="], } } diff --git a/dstack/kms/auth-simple/index.test.ts b/dstack/kms/auth-simple/index.test.ts index 0d5da07d0..32ad06772 100644 --- a/dstack/kms/auth-simple/index.test.ts +++ b/dstack/kms/auth-simple/index.test.ts @@ -9,7 +9,7 @@ import app from './index'; const TEST_CONFIG_PATH = './test-auth-config.json'; const baseBootInfo = { - attestationMode: 'dstack-tdx', + teeVariant: 'dstack-tdx', mrAggregated: '0xabc123', osImageHash: '0x1fbb0cf9cc6cfbf23d6b779776fabad2c5403d643badb9e5e238615e4960a78a', appId: '0xapp123', @@ -106,7 +106,7 @@ describe('auth-simple', () => { const sevSnpBootInfo = { ...baseBootInfo, - attestationMode: 'dstack-amd-sev-snp', + teeVariant: 'dstack-amd-sev-snp', tcbStatus: 'OutOfDate' }; const denied = await app.fetch(new Request('http://localhost/bootAuth/kms', { diff --git a/dstack/kms/auth-simple/index.ts b/dstack/kms/auth-simple/index.ts index b8d8c86c2..4a833d1ec 100644 --- a/dstack/kms/auth-simple/index.ts +++ b/dstack/kms/auth-simple/index.ts @@ -9,7 +9,7 @@ import { readFileSync, existsSync } from 'fs'; // zod schemas for validation - compatible with auth-eth implementation const BootInfoSchema = z.object({ - attestationMode: z.string().optional().default(''), + teeVariant: z.string().optional().default(''), mrAggregated: z.string().describe('aggregated MR measurement'), osImageHash: z.string().describe('OS Image hash'), appId: z.string().describe('application ID'), diff --git a/dstack/kms/auth-simple/package.json b/dstack/kms/auth-simple/package.json index 649026ccd..b7d86003e 100644 --- a/dstack/kms/auth-simple/package.json +++ b/dstack/kms/auth-simple/package.json @@ -15,7 +15,7 @@ "check": "bun run lint && bun run test:run" }, "dependencies": { - "hono": "4.12.25", + "hono": "4.13.5", "@hono/zod-validator": "0.2.2", "zod": "3.25.76" }, diff --git a/dstack/kms/dstack-app/builder/Dockerfile b/dstack/kms/dstack-app/builder/Dockerfile index 038324c3a..ec4e73f10 100644 --- a/dstack/kms/dstack-app/builder/Dockerfile +++ b/dstack/kms/dstack-app/builder/Dockerfile @@ -9,7 +9,7 @@ ARG DSTACK_REV ARG DSTACK_SRC_URL=https://github.com/Dstack-TEE/dstack.git WORKDIR /build RUN ./pin-packages.sh ./builder-pinned-packages.txt -RUN apt-get update && \ +RUN apt-get update -o APT::Update::Error-Mode=any && \ apt-get install -y --no-install-recommends \ git \ build-essential \ @@ -27,39 +27,24 @@ RUN cd repo/dstack && cargo build --release -p dstack-kms --target x86_64-unknow RUN echo "${DSTACK_REV}" > /build/.GIT_REV FROM debian:bookworm@sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe -COPY --from=build-shared /pin-packages.sh /config-qemu.sh /build/ -COPY ./shared/qemu-pinned-packages.txt /build/ +COPY --from=build-shared /pin-packages.sh /build/ +COPY ./shared/pinned-packages.txt /build/ WORKDIR /build -ARG QEMU_REV=dbcec07c0854bf873d346a09e87e4c993ccf2633 -RUN ./pin-packages.sh ./qemu-pinned-packages.txt && \ - apt-get update && \ +# reqwest is built against rustls-platform-verifier, which reads the system +# trust store and errors out when it finds none. The KMS builds its collateral +# and AMD KDS clients eagerly at startup, so a base image without CA +# certificates fails to boot before it serves anything. +RUN ./pin-packages.sh ./pinned-packages.txt && \ + apt-get update -o APT::Update::Error-Mode=any && \ apt-get install -y --no-install-recommends \ - git \ - libslirp-dev \ - python3-pip \ - ninja-build \ - pkg-config \ - libglib2.0-dev \ - python3-sphinx \ - python3-sphinx-rtd-theme \ - build-essential \ - flex \ - bison && \ - rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache -RUN git clone https://github.com/kvinwang/qemu-tdx.git --depth 1 --branch dstack-qemu-9.2.1 --single-branch && \ - cd qemu-tdx && git fetch --depth 1 origin ${QEMU_REV} && \ - git checkout ${QEMU_REV} && \ - ../config-qemu.sh ./build /usr/local && \ - cd build && \ - ninja && \ - strip qemu-system-x86_64 && \ - install -m 755 qemu-system-x86_64 /usr/local/bin/dstack-acpi-tables && \ - cd ../ && \ - install -d /usr/local/share/qemu && \ - install -m 644 pc-bios/efi-virtio.rom /usr/local/share/qemu/ && \ - install -m 644 pc-bios/kvmvapic.bin /usr/local/share/qemu/ && \ - install -m 644 pc-bios/linuxboot_dma.bin /usr/local/share/qemu/ && \ - cd .. && rm -rf qemu-tdx + ca-certificates \ + && rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache +WORKDIR / COPY --from=kms-builder /build/repo/dstack/target/x86_64-unknown-linux-musl/release/dstack-kms /usr/local/bin/dstack-kms COPY --from=kms-builder /build/.GIT_REV /etc/ +# The OCI metadata is generated once by build-lib.sh and injected here, as +# the config labels and as the manifest annotations, so the three views of +# the image can never disagree. +ARG IMAGE_METADATA +RUN mkdir -p /etc/dstack-kms && printf '%s\n' "${IMAGE_METADATA}" > /etc/dstack-kms/build-info CMD ["dstack-kms"] diff --git a/dstack/kms/dstack-app/builder/README.md b/dstack/kms/dstack-app/builder/README.md index b5387fa93..923b34ebc 100644 --- a/dstack/kms/dstack-app/builder/README.md +++ b/dstack/kms/dstack-app/builder/README.md @@ -6,11 +6,11 @@ This directory contains the necessary files to build and run the dstack-kms Dock The builder creates a Docker image that includes: - The dstack-kms service compiled from Rust source code -- Command line tool dstack-acpi-tables for generating ACPI tables for dstack CVM +- Pure-Rust ACPI measurement support built into the dstack-kms binary ## Prerequisites -- Docker with BuildKit support (v20.10.0+) +- Docker with Buildx v0.13.0+ (the script creates a BuildKit v0.20.2 builder) - Git ## Building the Image @@ -18,7 +18,7 @@ The builder creates a Docker image that includes: To build the KMS Docker image, use the provided `build-image.sh` script: ```bash -./build-image.sh <image-name>[:<tag>] +./build-image.sh <image-name>[:<tag>]... ``` For example: @@ -26,6 +26,63 @@ For example: ./build-image.sh kvin/kms ``` +Optional environment variables: + +| Variable | Purpose | +| --- | --- | +| `GIT_REV` | Revision to build (default `HEAD`) | +| `IMAGE_VERSION` | Version recorded in the image metadata (default `dev`) | +| `IMAGE_SOURCE_URL` | Repository URL recorded in the image metadata | +| `NO_CACHE` | Set to any value to build without the layer cache | +| `OCI_TAR` | Also write an OCI archive here, for digest comparison | +| `METADATA_FILE` | Write the validated OCI manifest digest and build metadata here | +| `PUSH` | Set to any value to push the tags instead of only loading them | + +Publication and OCI export happen only after both package lists pass validation. +`NO_CACHE` applies to the validation builds; export then reuses their cached result. +Manual release workflows require an existing component release tag and check out +that tag, rather than building the branch selected in the workflow UI. + +## Reproducing a released image + +Release CI runs this same script, so a published image can be rebuilt and +checked digest-for-digest. Use a clean checkout of the release commit so the +Dockerfile, package lists, shared scripts and copied files also match the release. +From the repository root (fetch the tag first if it is not available locally): + +```bash +git switch --detach "kms-v0.6.0^{commit}" +cd dstack/kms/dstack-app/builder + +GIT_REV=HEAD \ +IMAGE_VERSION=0.6.0 \ +IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack \ +OCI_TAR=/tmp/kms.oci.tar \ + ./build-image.sh dstacktee/dstack-kms:0.6.0 + +python3 -c 'import json,tarfile;t=tarfile.open("/tmp/kms.oci.tar");print(json.load(t.extractfile("index.json"))["manifests"][0]["digest"])' +``` + +`IMAGE_VERSION` is part of the image metadata, so it must match the release for +the digests to match. The printed digest is what the registry reports for +`dstacktee/dstack-kms:0.6.0`; compare it with: + +```bash +docker buildx imagetools inspect dstacktee/dstack-kms:0.6.0 --format '{{.Manifest.Digest}}' +``` + +This is the digest that `deploy-to-vmm.sh` pins in `KMS_IMAGE`, and that in turn +feeds the compose hash registered on chain. + +## Image metadata + +The image carries its provenance as OCI metadata in three places, all generated +from one definition in `dstack/build/shared/build-lib.sh`: + +- config labels — `docker inspect -f '{{json .Config.Labels}}' <image>` +- manifest annotations — `docker buildx imagetools inspect <image>` +- `/etc/dstack-kms/build-info` inside the image, readable from within the CVM + ## Running the Built Image ### Using Docker Compose diff --git a/dstack/kms/dstack-app/builder/build-image.sh b/dstack/kms/dstack-app/builder/build-image.sh index b92c9adcb..c04c54f1d 100755 --- a/dstack/kms/dstack-app/builder/build-image.sh +++ b/dstack/kms/dstack-app/builder/build-image.sh @@ -11,26 +11,31 @@ REPO_ROOT=$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel) CONTEXT_DIR="$SCRIPT_DIR" SHARED_DIR="$SCRIPT_DIR/shared" DOCKERFILE="$SCRIPT_DIR/Dockerfile" +export CONTEXT_DIR DOCKERFILE +# shellcheck source=/dev/null source "$REPO_ROOT/dstack/build/shared/build-lib.sh" -NAME=${1:-} -if [ -z "$NAME" ]; then - echo "Usage: $0 <image-name>[:<tag>]" >&2 +if [ "$#" -eq 0 ]; then + echo "Usage: $0 <image-name>[:<tag>]..." >&2 exit 1 fi +TAGS=$(printf '%s\n' "$@") NO_CACHE=${NO_CACHE:-} GIT_REV=${GIT_REV:-HEAD} -GIT_REV=$(git -C "$REPO_ROOT" rev-parse "$GIT_REV") +GIT_REV=$(git -C "$REPO_ROOT" rev-parse --verify "${GIT_REV}^{commit}") DSTACK_SRC_URL=${DSTACK_SRC_URL:-https://github.com/Dstack-TEE/dstack.git} ensure_buildkit touch "$SHARED_DIR/builder-pinned-packages.txt" -touch "$SHARED_DIR/qemu-pinned-packages.txt" +touch "$SHARED_DIR/pinned-packages.txt" -docker_build "$NAME" "" "$SHARED_DIR/qemu-pinned-packages.txt" -docker_build "kms-builder-temp" "kms-builder" "$SHARED_DIR/builder-pinned-packages.txt" +METADATA=$(image_metadata \ + "dstack-kms" \ + "Key management service for dstack confidential applications" \ + "dstack/kms" \ + "dstack/kms/README.md") -check_clean_tree "$SHARED_DIR" +build_component "$TAGS" "kms-builder-temp" "kms-builder" "$SHARED_DIR" "$METADATA" diff --git a/dstack/kms/dstack-app/builder/shared/pinned-packages.txt b/dstack/kms/dstack-app/builder/shared/pinned-packages.txt new file mode 100644 index 000000000..e966045f9 --- /dev/null +++ b/dstack/kms/dstack-app/builder/shared/pinned-packages.txt @@ -0,0 +1,91 @@ +adduser=3.134 +apt=2.6.1 +base-files=12.4+deb12u11 +base-passwd=3.6.1 +bash=5.2.15-2+b8 +bsdutils=1:2.38.1-5+deb12u3 +ca-certificates=20230311+deb12u1 +coreutils=9.1-1 +dash=0.5.12-2 +debconf=1.5.82 +debian-archive-keyring=2023.3+deb12u2 +debianutils=5.7-0.5~deb12u1 +diffutils=1:3.8-4 +dpkg=1.21.22 +e2fsprogs=1.47.0-2 +findutils=4.9.0-4 +gcc-12-base:amd64=12.2.0-14+deb12u1 +gpgv=2.2.40-1.1 +grep=3.8-5 +gzip=1.12-1 +hostname=3.23+nmu1 +init-system-helpers=1.65.2 +libacl1:amd64=2.3.1-3 +libapt-pkg6.0:amd64=2.6.1 +libattr1:amd64=1:2.5.1-4 +libaudit-common=1:3.0.9-1 +libaudit1:amd64=1:3.0.9-1 +libblkid1:amd64=2.38.1-5+deb12u3 +libbz2-1.0:amd64=1.0.8-5+b1 +libc-bin=2.36-9+deb12u10 +libc6:amd64=2.36-9+deb12u10 +libcap-ng0:amd64=0.8.3-1+b3 +libcap2:amd64=1:2.66-4+deb12u1 +libcom-err2:amd64=1.47.0-2 +libcrypt1:amd64=1:4.4.33-2 +libdb5.3:amd64=5.3.28+dfsg2-1 +libdebconfclient0:amd64=0.270 +libext2fs2:amd64=1.47.0-2 +libffi8:amd64=3.4.4-1 +libgcc-s1:amd64=12.2.0-14+deb12u1 +libgcrypt20:amd64=1.10.1-3 +libgmp10:amd64=2:6.2.1+dfsg1-1.1 +libgnutls30:amd64=3.7.9-2+deb12u4 +libgpg-error0:amd64=1.46-1 +libhogweed6:amd64=3.8.1-2 +libidn2-0:amd64=2.3.3-1+b1 +liblz4-1:amd64=1.9.4-1 +liblzma5:amd64=5.4.1-1 +libmd0:amd64=1.0.4-2 +libmount1:amd64=2.38.1-5+deb12u3 +libnettle8:amd64=3.8.1-2 +libp11-kit0:amd64=0.24.1-2 +libpam-modules-bin=1.5.2-6+deb12u1 +libpam-modules:amd64=1.5.2-6+deb12u1 +libpam-runtime=1.5.2-6+deb12u1 +libpam0g:amd64=1.5.2-6+deb12u1 +libpcre2-8-0:amd64=10.42-1 +libseccomp2:amd64=2.5.4-1+deb12u1 +libselinux1:amd64=3.4-1+b6 +libsemanage-common=3.4-1 +libsemanage2:amd64=3.4-1+b5 +libsepol2:amd64=3.4-2.1 +libsmartcols1:amd64=2.38.1-5+deb12u3 +libss2:amd64=1.47.0-2 +libssl3:amd64=3.0.18-1~deb12u2 +libstdc++6:amd64=12.2.0-14+deb12u1 +libsystemd0:amd64=252.38-1~deb12u1 +libtasn1-6:amd64=4.19.0-2+deb12u1 +libtinfo6:amd64=6.4-4 +libudev1:amd64=252.38-1~deb12u1 +libunistring2:amd64=1.0-2 +libuuid1:amd64=2.38.1-5+deb12u3 +libxxhash0:amd64=0.8.1-1 +libzstd1:amd64=1.5.4+dfsg2-5 +login=1:4.13+dfsg1-1+deb12u1 +logsave=1.47.0-2 +mawk=1.3.4.20200120-3.1 +mount=2.38.1-5+deb12u3 +ncurses-base=6.4-4 +ncurses-bin=6.4-4 +openssl=3.0.18-1~deb12u2 +passwd=1:4.13+dfsg1-1+deb12u1 +perl-base=5.36.0-7+deb12u2 +sed=4.9-1 +sysvinit-utils=3.06-4 +tar=1.34+dfsg-1.2+deb12u1 +tzdata=2025b-0+deb12u1 +usr-is-merged=37~deb12u1 +util-linux-extra=2.38.1-5+deb12u3 +util-linux=2.38.1-5+deb12u3 +zlib1g:amd64=1:1.2.13.dfsg-1 diff --git a/dstack/kms/dstack-app/builder/shared/qemu-pinned-packages.txt b/dstack/kms/dstack-app/builder/shared/qemu-pinned-packages.txt deleted file mode 100644 index 8cb00cf86..000000000 --- a/dstack/kms/dstack-app/builder/shared/qemu-pinned-packages.txt +++ /dev/null @@ -1,236 +0,0 @@ -adduser=3.134 -apt=2.6.1 -base-files=12.4+deb12u11 -base-passwd=3.6.1 -bash=5.2.15-2+b8 -binutils-common:amd64=2.40-2 -binutils-x86-64-linux-gnu=2.40-2 -binutils=2.40-2 -bison=2:3.8.2+dfsg-1+b1 -bsdutils=1:2.38.1-5+deb12u3 -build-essential=12.9 -bzip2=1.0.8-5+b1 -ca-certificates=20230311+deb12u1 -coreutils=9.1-1 -cpp-12=12.2.0-14+deb12u1 -cpp=4:12.2.0-3 -dash=0.5.12-2 -debconf=1.5.82 -debian-archive-keyring=2023.3+deb12u2 -debianutils=5.7-0.5~deb12u1 -diffutils=1:3.8-4 -docutils-common=0.19+dfsg-6 -dpkg-dev=1.21.22 -dpkg=1.21.22 -e2fsprogs=1.47.0-2 -findutils=4.9.0-4 -flex=2.6.4-8.2 -fonts-font-awesome=5.0.10+really4.7.0~dfsg-4.1 -fonts-lato=2.0-2.1 -g++-12=12.2.0-14+deb12u1 -g++=4:12.2.0-3 -gcc-12-base:amd64=12.2.0-14+deb12u1 -gcc-12=12.2.0-14+deb12u1 -gcc=4:12.2.0-3 -git-man=1:2.39.5-0+deb12u3 -git=1:2.39.5-0+deb12u3 -gpgv=2.2.40-1.1 -grep=3.8-5 -gzip=1.12-1 -hostname=3.23+nmu1 -init-system-helpers=1.65.2 -libacl1:amd64=2.3.1-3 -libapt-pkg6.0:amd64=2.6.1 -libasan8:amd64=12.2.0-14+deb12u1 -libatomic1:amd64=12.2.0-14+deb12u1 -libattr1:amd64=1:2.5.1-4 -libaudit-common=1:3.0.9-1 -libaudit1:amd64=1:3.0.9-1 -libbinutils:amd64=2.40-2 -libblkid-dev:amd64=2.38.1-5+deb12u3 -libblkid1:amd64=2.38.1-5+deb12u3 -libbrotli1:amd64=1.0.9-2+b6 -libbz2-1.0:amd64=1.0.8-5+b1 -libc-bin=2.36-9+deb12u13 -libc-dev-bin=2.36-9+deb12u13 -libc6-dev:amd64=2.36-9+deb12u13 -libc6:amd64=2.36-9+deb12u13 -libcap-ng0:amd64=0.8.3-1+b3 -libcap2:amd64=1:2.66-4+deb12u1 -libcc1-0:amd64=12.2.0-14+deb12u1 -libcom-err2:amd64=1.47.0-2 -libcrypt-dev:amd64=1:4.4.33-2 -libcrypt1:amd64=1:4.4.33-2 -libctf-nobfd0:amd64=2.40-2 -libctf0:amd64=2.40-2 -libcurl3-gnutls:amd64=7.88.1-10+deb12u14 -libdb5.3:amd64=5.3.28+dfsg2-1 -libdebconfclient0:amd64=0.270 -libdpkg-perl=1.21.22 -libelf1:amd64=0.188-2.1 -liberror-perl=0.17029-2 -libexpat1:amd64=2.5.0-1+deb12u2 -libext2fs2:amd64=1.47.0-2 -libffi-dev:amd64=3.4.4-1 -libffi8:amd64=3.4.4-1 -libgcc-12-dev:amd64=12.2.0-14+deb12u1 -libgcc-s1:amd64=12.2.0-14+deb12u1 -libgcrypt20:amd64=1.10.1-3 -libgdbm-compat4:amd64=1.23-3 -libgdbm6:amd64=1.23-3 -libglib2.0-0:amd64=2.74.6-2+deb12u8 -libglib2.0-bin=2.74.6-2+deb12u8 -libglib2.0-data=2.74.6-2+deb12u8 -libglib2.0-dev-bin=2.74.6-2+deb12u8 -libglib2.0-dev:amd64=2.74.6-2+deb12u8 -libgmp10:amd64=2:6.2.1+dfsg1-1.1 -libgnutls30:amd64=3.7.9-2+deb12u4 -libgomp1:amd64=12.2.0-14+deb12u1 -libgpg-error0:amd64=1.46-1 -libgprofng0:amd64=2.40-2 -libgssapi-krb5-2:amd64=1.20.1-2+deb12u4 -libhogweed6:amd64=3.8.1-2 -libidn2-0:amd64=2.3.3-1+b1 -libisl23:amd64=0.25-1.1 -libitm1:amd64=12.2.0-14+deb12u1 -libjansson4:amd64=2.14-2 -libjs-jquery=3.6.1+dfsg+~3.5.14-1 -libjs-sphinxdoc=5.3.0-4 -libjs-underscore=1.13.4~dfsg+~1.11.4-3 -libjson-perl=4.10000-1 -libk5crypto3:amd64=1.20.1-2+deb12u4 -libkeyutils1:amd64=1.6.3-2 -libkrb5-3:amd64=1.20.1-2+deb12u4 -libkrb5support0:amd64=1.20.1-2+deb12u4 -libldap-2.5-0:amd64=2.5.13+dfsg-5 -liblsan0:amd64=12.2.0-14+deb12u1 -liblz4-1:amd64=1.9.4-1 -liblzma5:amd64=5.4.1-1 -libmd0:amd64=1.0.4-2 -libmount-dev:amd64=2.38.1-5+deb12u3 -libmount1:amd64=2.38.1-5+deb12u3 -libmpc3:amd64=1.3.1-1 -libmpfr6:amd64=4.2.0-1 -libncursesw6:amd64=6.4-4 -libnettle8:amd64=3.8.1-2 -libnghttp2-14:amd64=1.52.0-1+deb12u2 -libnsl-dev:amd64=1.3.0-2 -libnsl2:amd64=1.3.0-2 -libp11-kit0:amd64=0.24.1-2 -libpam-modules-bin=1.5.2-6+deb12u1 -libpam-modules:amd64=1.5.2-6+deb12u1 -libpam-runtime=1.5.2-6+deb12u1 -libpam0g:amd64=1.5.2-6+deb12u1 -libpcre2-16-0:amd64=10.42-1 -libpcre2-32-0:amd64=10.42-1 -libpcre2-8-0:amd64=10.42-1 -libpcre2-dev:amd64=10.42-1 -libpcre2-posix3:amd64=10.42-1 -libperl5.36:amd64=5.36.0-7+deb12u3 -libpkgconf3:amd64=1.8.1-1 -libpsl5:amd64=0.21.2-1 -libpython3-stdlib:amd64=3.11.2-1+b1 -libpython3.11-minimal:amd64=3.11.2-6+deb12u6 -libpython3.11-stdlib:amd64=3.11.2-6+deb12u6 -libquadmath0:amd64=12.2.0-14+deb12u1 -libreadline8:amd64=8.2-1.3 -librtmp1:amd64=2.4+20151223.gitfa8646d.1-2+b2 -libsasl2-2:amd64=2.1.28+dfsg-10 -libsasl2-modules-db:amd64=2.1.28+dfsg-10 -libseccomp2:amd64=2.5.4-1+deb12u1 -libselinux1-dev:amd64=3.4-1+b6 -libselinux1:amd64=3.4-1+b6 -libsemanage-common=3.4-1 -libsemanage2:amd64=3.4-1+b5 -libsepol-dev:amd64=3.4-2.1 -libsepol2:amd64=3.4-2.1 -libslirp-dev:amd64=4.7.0-1 -libslirp0:amd64=4.7.0-1 -libsmartcols1:amd64=2.38.1-5+deb12u3 -libsqlite3-0:amd64=3.40.1-2+deb12u2 -libss2:amd64=1.47.0-2 -libssh2-1:amd64=1.10.0-3+b1 -libssl3:amd64=3.0.18-1~deb12u2 -libstdc++-12-dev:amd64=12.2.0-14+deb12u1 -libstdc++6:amd64=12.2.0-14+deb12u1 -libsystemd0:amd64=252.38-1~deb12u1 -libtasn1-6:amd64=4.19.0-2+deb12u1 -libtinfo6:amd64=6.4-4 -libtirpc-common=1.3.3+ds-1 -libtirpc-dev:amd64=1.3.3+ds-1 -libtirpc3:amd64=1.3.3+ds-1 -libtsan2:amd64=12.2.0-14+deb12u1 -libubsan1:amd64=12.2.0-14+deb12u1 -libudev1:amd64=252.38-1~deb12u1 -libunistring2:amd64=1.0-2 -libuuid1:amd64=2.38.1-5+deb12u3 -libxxhash0:amd64=0.8.1-1 -libzstd1:amd64=1.5.4+dfsg2-5 -linux-libc-dev:amd64=6.1.164-1 -login=1:4.13+dfsg1-1+deb12u1 -logsave=1.47.0-2 -m4=1.4.19-3 -make=4.3-4.1 -mawk=1.3.4.20200120-3.1 -media-types=10.0.0 -mount=2.38.1-5+deb12u3 -ncurses-base=6.4-4 -ncurses-bin=6.4-4 -ninja-build=1.11.1-2~deb12u1 -openssl=3.0.18-1~deb12u2 -passwd=1:4.13+dfsg1-1+deb12u1 -patch=2.7.6-7 -perl-base=5.36.0-7+deb12u3 -perl-modules-5.36=5.36.0-7+deb12u3 -perl=5.36.0-7+deb12u3 -pkg-config:amd64=1.8.1-1 -pkgconf-bin=1.8.1-1 -pkgconf:amd64=1.8.1-1 -python-babel-localedata=2.10.3-1 -python3-alabaster=0.7.12-1 -python3-babel=2.10.3-1 -python3-certifi=2022.9.24-1 -python3-chardet=5.1.0+dfsg-2 -python3-charset-normalizer=3.0.1-2 -python3-distutils=3.11.2-3 -python3-docutils=0.19+dfsg-6 -python3-idna=3.3-1+deb12u1 -python3-imagesize=1.4.1-1 -python3-jinja2=3.1.2-1+deb12u3 -python3-lib2to3=3.11.2-3 -python3-markupsafe=2.1.2-1+b1 -python3-minimal=3.11.2-1+b1 -python3-packaging=23.0-1 -python3-pip=23.0.1+dfsg-1 -python3-pkg-resources=66.1.1-1+deb12u2 -python3-pygments=2.14.0+dfsg-1 -python3-requests=2.28.1+dfsg-1 -python3-roman=3.3-3 -python3-setuptools=66.1.1-1+deb12u2 -python3-six=1.16.0-4 -python3-snowballstemmer=2.2.0-2 -python3-sphinx-rtd-theme=1.2.0+dfsg-1 -python3-sphinx=5.3.0-4 -python3-tz=2022.7.1-4 -python3-urllib3=1.26.12-1+deb12u3 -python3-wheel=0.38.4-2 -python3.11-minimal=3.11.2-6+deb12u6 -python3.11=3.11.2-6+deb12u6 -python3=3.11.2-1+b1 -readline-common=8.2-1.3 -rpcsvc-proto=1.4.3-1 -sed=4.9-1 -sgml-base=1.31 -sphinx-common=5.3.0-4 -sphinx-rtd-theme-common=1.2.0+dfsg-1 -sysvinit-utils=3.06-4 -tar=1.34+dfsg-1.2+deb12u1 -tzdata=2025b-0+deb12u1 -usr-is-merged=37~deb12u1 -util-linux-extra=2.38.1-5+deb12u3 -util-linux=2.38.1-5+deb12u3 -uuid-dev:amd64=2.38.1-5+deb12u3 -xml-core=0.18+nmu1 -xz-utils=5.4.1-1 -zlib1g-dev:amd64=1:1.2.13.dfsg-1 -zlib1g:amd64=1:1.2.13.dfsg-1 diff --git a/dstack/kms/dstack-app/compose-dev.yaml b/dstack/kms/dstack-app/compose-dev.yaml index c216496ef..a642b3218 100644 --- a/dstack/kms/dstack-app/compose-dev.yaml +++ b/dstack/kms/dstack-app/compose-dev.yaml @@ -8,17 +8,17 @@ services: build: context: . dockerfile_inline: | - FROM node:18-alpine@sha256:06f7bbbcec00dd10c21a3a0962609600159601b5004d84aff142977b449168e9 + FROM node:20-bookworm-slim@sha256:2cf067cfed83d5ea958367df9f966191a942351a2df77d6f0193e162b5febfc0 WORKDIR /app - RUN apk add --no-cache git + RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates && rm -rf /var/lib/apt/lists/* RUN git clone ${GIT_REPOSITORY} && \ cd dstack && \ git checkout ${GIT_REV} WORKDIR /app/dstack/dstack/kms/auth-eth - RUN npm install - RUN npx tsc --project tsconfig.json - CMD node dist/src/main.js + RUN npm ci + RUN npm run build + CMD node dist/main.js environment: - HOST=0.0.0.0 - PORT=8000 @@ -57,13 +57,8 @@ configs: key = "/kms/certs/rpc.key" certs = "/kms/certs/rpc.crt" - [rpc.tls.mutual] - ca_certs = "/kms/certs/tmp-ca.crt" - mandatory = false - [core] cert_dir = "/kms/certs" - admin_token_hash = "${ADMIN_TOKEN_HASH}" [core.image] verify = true diff --git a/dstack/kms/dstack-app/compose-simple.yaml b/dstack/kms/dstack-app/compose-simple.yaml index 522ad95d0..711b06cad 100644 --- a/dstack/kms/dstack-app/compose-simple.yaml +++ b/dstack/kms/dstack-app/compose-simple.yaml @@ -33,13 +33,8 @@ configs: key = "/kms/certs/rpc.key" certs = "/kms/certs/rpc.crt" - [rpc.tls.mutual] - ca_certs = "/kms/certs/tmp-ca.crt" - mandatory = false - [core] cert_dir = "/kms/certs" - admin_token_hash = "${ADMIN_TOKEN_HASH}" [core.image] verify = true diff --git a/dstack/kms/dstack-app/docker-compose.yaml b/dstack/kms/dstack-app/docker-compose.yaml index 3d8a18f5d..45dfe7fa8 100644 --- a/dstack/kms/dstack-app/docker-compose.yaml +++ b/dstack/kms/dstack-app/docker-compose.yaml @@ -10,7 +10,7 @@ services: dockerfile_inline: | FROM rust:1.92.0@sha256:48851a839d6a67370c9dbe0e709bedc138e3e404b161c5233aedcf2b717366e4 WORKDIR /app - RUN apk add --no-cache git build-base openssl-dev protobuf protobuf-dev perl + RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates && rm -rf /var/lib/apt/lists/* build-base openssl-dev protobuf protobuf-dev perl RUN git clone https://github.com/a16z/helios && \ cd helios && \ git checkout 5c61864a167c16141a9a12b976c0e9398b332f07 @@ -32,18 +32,16 @@ services: build: context: . dockerfile_inline: | - FROM node:18-alpine@sha256:06f7bbbcec00dd10c21a3a0962609600159601b5004d84aff142977b449168e9 + FROM node:20-bookworm-slim@sha256:2cf067cfed83d5ea958367df9f966191a942351a2df77d6f0193e162b5febfc0 WORKDIR /app - RUN apk add --no-cache git + RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates && rm -rf /var/lib/apt/lists/* RUN git clone https://github.com/Dstack-TEE/dstack.git && \ cd dstack && \ git checkout 78057c975fe4b9e21f557fb888d72eeecfb21178 WORKDIR /app/dstack/kms/auth-eth - RUN npm install && \ - npx hardhat typechain && \ - npx tsc --project tsconfig.json - CMD node dist/src/main.js + RUN npm ci && npm run build + CMD node dist/main.js environment: - HOST=0.0.0.0 - PORT=8000 diff --git a/dstack/kms/dstack-app/entrypoint.sh b/dstack/kms/dstack-app/entrypoint.sh index 781c31fa5..52d14b8c6 100755 --- a/dstack/kms/dstack-app/entrypoint.sh +++ b/dstack/kms/dstack-app/entrypoint.sh @@ -8,9 +8,6 @@ set -e cat <<EOF > ./kms.toml -[core] -admin_token_hash = "${ADMIN_TOKEN_HASH}" - [core.image] verify = ${VERIFY_IMAGE} cache_dir = "./images" diff --git a/dstack/kms/kms.toml b/dstack/kms/kms.toml index 2e9b9eb13..5beb28d80 100644 --- a/dstack/kms/kms.toml +++ b/dstack/kms/kms.toml @@ -18,24 +18,25 @@ port = 8000 key = "/etc/kms/certs/rpc.key" certs = "/etc/kms/certs/rpc.crt" -[rpc.tls.mutual] -ca_certs = "/etc/kms/certs/tmp-ca.crt" -mandatory = false +# No `[rpc.tls.mutual]`: client certificates are verified by the attestation they +# carry, not by an issuer CA (see ra-rpc's ratls_client_verifier). Leaving a +# `mutual` section here is harmless — it is no longer the trust anchor. [core] cert_dir = "/etc/kms/certs" subject_postfix = ".dstack" -admin_token_hash = "" site_name = "" # Whether trusted RPCs require the KMS to first attest itself to its own # auth API. Defaults to true (strict). Set to false ONLY when running KMS # outside a TEE (e.g. local dev/testing) where the local guest agent socket # is unavailable. enforce_self_authorization = true -# Optional AMD KDS-compatible base URL for SEV-SNP collateral fetches. -# Leave empty to use the built-in default. -amd_kds_base_url = "" - +# Whether the KMS embeds an attestation in its own RPC certificate. Set false +# only for local dev/testing where the KMS runs outside a TEE. This narrows what +# the KMS asserts about itself and relaxes no verification: quotes presented to +# the KMS are still fully checked, and another KMS refuses to onboard from an +# unattested one. +attest_rpc_cert = true # AMD SEV-SNP key/cert release remains disabled unless this local KMS gate is # explicitly enabled. External auth policy must still allow the verified # BootInfo before any sensitive material is returned. @@ -47,12 +48,48 @@ sev_snp_key_release = false # auth policy must still allow the verified BootInfo. aws_nitro_tpm_key_release = false +# Optional attestation trust-anchor files. Omitted entries use vendor roots. +[core.attestation] +insecure_allow_external_trust_anchors = false + +[core.attestation.urls] +# pccs = "https://pccs.phala.network" +# amd_kds = "https://kdsintf.amd.com/vcek/v1" + +[core.attestation.root_ca] +# tdx = "/etc/dstack/roots/intel-sgx-root-ca.der" +# gcp_tpm = "/etc/dstack/roots/gcp-tpm-root-ca.pem" +# aws_nitro_enclave = "/etc/dstack/roots/aws-nitro-enclave-root-ca.pem" +# aws_nitro_tpm = "/etc/dstack/roots/aws-nitro-tpm-root-ca.pem" +# sev_snp_milan = "/etc/dstack/roots/amd-milan-ark.pem" +# sev_snp_genoa = "/etc/dstack/roots/amd-genoa-ark.pem" +# sev_snp_turin = "/etc/dstack/roots/amd-turin-ark.pem" + [core.image] verify = true cache_dir = "/usr/share/dstack/images" download_url = "http://localhost:8000/{OS_IMAGE_HASH}.tar.gz" download_timeout = "2m" +# Admin API: serves operator RPCs (e.g. ClearImageCache) on a dedicated +# listener behind the shared HTTP authenticator. Disabled by default; when +# enabled it fails closed unless auth_token or htpasswd_file is set. +[core.admin] +enabled = false +# Bind address/port for the admin listener (keep it off the public network). +address = "127.0.0.1" +port = 8001 +# Shared admin token required by every admin RPC. Can also be supplied via the +# `DSTACK_KMS_ADMIN_TOKEN` or `ADMIN_API_TOKEN` env vars. Clients send it as +# `Authorization: Bearer <token>` or the `X-Admin-Token: <token>` header. +# Required unless insecure_no_auth = true. Generate with: openssl rand -hex 32 +auth_token = "" +# Optional Apache bcrypt htpasswd file (create with `htpasswd -B -c file admin`). +htpasswd_file = "" +# Development/testing escape hatch only. Never enable this on an admin +# interface that is reachable from the network. +insecure_no_auth = false + [core.metrics] # Expose unauthenticated Prometheus metrics on /metrics. # Disable this if the KMS RPC listener is not protected by network policy diff --git a/dstack/kms/rpc/proto/kms_rpc.proto b/dstack/kms/rpc/proto/kms_rpc.proto index 9ef55b98e..e491d9335 100644 --- a/dstack/kms/rpc/proto/kms_rpc.proto +++ b/dstack/kms/rpc/proto/kms_rpc.proto @@ -74,6 +74,11 @@ message KmsKeyResponse { repeated KmsKeys keys = 2; } +// See the note on `GetTempCaCert`. +// +// `temp_ca_key` is a CA private key handed to any caller, so it authenticates +// nobody. It exists because the KMS pinned this CA for mutual TLS, which left a +// self-issued RA-TLS certificate with nothing to chain to. message GetTempCaCertResponse { string temp_ca_cert = 1; string temp_ca_key = 2; @@ -101,18 +106,36 @@ service KMS { rpc GetAppEnvEncryptPubKey(AppId) returns (PublicKeyResponse); // Request the KMS instance metadata for use as a probe and health check. rpc GetMeta(google.protobuf.Empty) returns (GetMetaResponse); - // Request the temporary CA certificate and key + // Request the temporary CA certificate and key. + // + // Both current callers - guests at boot, and KMS-to-KMS onboarding - fetch this CA + // and mint their client certificate from it, because the KMS used to pin the CA for + // mutual TLS and a self-issued certificate had nothing to chain to. + // + // That pin is gone: the KMS now verifies client certificates by the attestation + // they carry and ignores the issuer, so a self-issued certificate would be accepted + // on the same terms. Neither caller has been migrated yet, so this RPC still has to + // work; it is scheduled for removal once both are. + // + // Do not build new callers on it. A caller that only needs the KMS root CA can read + // `ca_cert` from `GetMeta` instead. rpc GetTempCaCert(google.protobuf.Empty) returns (GetTempCaCertResponse); // Sign a certificate rpc SignCert(SignCertRequest) returns (SignCertResponse); +} + +// The KMS admin RPC service. Served on a separate admin listener +// (`[core.admin]`) behind the shared HTTP authenticator, so the credential +// travels in the `Authorization`/`X-Admin-Token` header rather than the +// request body. +service Admin { // Clear the image cache rpc ClearImageCache(ClearImageCacheRequest) returns (google.protobuf.Empty); } message ClearImageCacheRequest { - string token = 1; - string image_hash = 2; - string config_hash = 3; + string image_hash = 1; + string config_hash = 2; } message BootstrapRequest { @@ -143,8 +166,8 @@ message AttestationInfoResponse { bytes mr_aggregated = 2; // OS image hash bytes os_image_hash = 3; - // Attestation mode (e.g. "dstack-tdx", "dstack-gcp-tdx") - string attestation_mode = 4; + // TEE variant (e.g. "dstack-tdx", "dstack-gcp-tdx") + string tee_variant = 4; // Custom site name for display string site_name = 5; // Ethereum RPC URL from auth API diff --git a/dstack/kms/src/admin_auth.rs b/dstack/kms/src/admin_auth.rs new file mode 100644 index 000000000..22fe33bd6 --- /dev/null +++ b/dstack/kms/src/admin_auth.rs @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! KMS adapter for the shared API authenticator, guarding the admin listener. +//! +//! Unlike the KMS public RPC surface (which authenticates callers by RA-TLS +//! attestation), the admin API is reached by operators/tooling, so it uses the +//! same shared-token/htpasswd HTTP mechanism as the VMM and gateway. The token +//! is configured via `core.admin.auth_token`, or the `DSTACK_KMS_ADMIN_TOKEN` / +//! `ADMIN_API_TOKEN` environment variables. + +use anyhow::{bail, Result}; +use dstack_api_auth::{Authenticator, HttpAuthConfig, HttpAuthFairing}; +use rocket::Route; + +use crate::config::AdminConfig; + +const ENV_ADMIN_TOKEN: &str = "DSTACK_KMS_ADMIN_TOKEN"; +const ENV_ADMIN_TOKEN_COMPAT: &str = "ADMIN_API_TOKEN"; + +pub struct AdminAuthFairing(HttpAuthFairing); + +impl AdminAuthFairing { + pub fn from_config(config: &AdminConfig) -> Result<Self> { + if config.insecure_no_auth { + return Ok(Self(HttpAuthFairing::new( + Authenticator::disabled(), + http_config(), + ))); + } + let token = if !config.auth_token.is_empty() { + config.auth_token.trim().to_owned() + } else { + std::env::var(ENV_ADMIN_TOKEN) + .or_else(|_| std::env::var(ENV_ADMIN_TOKEN_COMPAT)) + .unwrap_or_default() + .trim() + .to_owned() + }; + if token.is_empty() && config.htpasswd_file.as_os_str().is_empty() { + bail!( + "admin API is enabled but neither auth_token nor htpasswd_file is configured; \ + set core.admin.auth_token, {ENV_ADMIN_TOKEN}, {ENV_ADMIN_TOKEN_COMPAT}, \ + core.admin.htpasswd_file, or insecure_no_auth = true (testing only)" + ); + } + let mut auth = Authenticator::from_tokens([token]); + if !config.htpasswd_file.as_os_str().is_empty() { + auth = auth.with_htpasswd_file(&config.htpasswd_file)?; + } + Ok(Self(HttpAuthFairing::new(auth, http_config()))) + } +} + +fn http_config() -> HttpAuthConfig { + HttpAuthConfig { + realm: "dstack-kms admin".into(), + token_header: Some("X-Admin-Token".into()), + allow_get_query_token: false, + } +} + +#[rocket::async_trait] +impl rocket::fairing::Fairing for AdminAuthFairing { + fn info(&self) -> rocket::fairing::Info { + self.0.info() + } + async fn on_request(&self, req: &mut rocket::Request<'_>, data: &mut rocket::Data<'_>) { + self.0.on_request(req, data).await + } +} + +pub fn routes() -> Vec<Route> { + dstack_api_auth::routes() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn enabled_without_credentials_fails_closed() { + // the token can also come from the environment; make sure neither var is + // set so this exercises the missing-credential path. + std::env::remove_var(ENV_ADMIN_TOKEN); + std::env::remove_var(ENV_ADMIN_TOKEN_COMPAT); + let cfg = AdminConfig { + enabled: true, + ..Default::default() + }; + assert!(AdminAuthFairing::from_config(&cfg).is_err()); + } + + #[test] + fn insecure_no_auth_is_allowed() { + let cfg = AdminConfig { + enabled: true, + insecure_no_auth: true, + ..Default::default() + }; + assert!(AdminAuthFairing::from_config(&cfg).is_ok()); + } + + #[test] + fn configured_token_builds() { + let cfg = AdminConfig { + enabled: true, + auth_token: "secret".into(), + ..Default::default() + }; + assert!(AdminAuthFairing::from_config(&cfg).is_ok()); + } +} diff --git a/dstack/kms/src/admin_service.rs b/dstack/kms/src/admin_service.rs new file mode 100644 index 000000000..1b8bec309 --- /dev/null +++ b/dstack/kms/src/admin_service.rs @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! KMS admin RPC service, served on the `[core.admin]` listener behind the +//! shared HTTP authenticator. Callers are authenticated by the admin token / +//! htpasswd via `Authorization`/`X-Admin-Token`, so the handlers do no token +//! checks of their own. + +use anyhow::Result; +use dstack_kms_rpc::{ + admin_server::{AdminRpc, AdminServer}, + ClearImageCacheRequest, +}; +use ra_rpc::{CallContext, RpcCall}; + +use crate::main_service::KmsState; + +pub struct AdminRpcHandler { + state: KmsState, +} + +impl AdminRpc for AdminRpcHandler { + async fn clear_image_cache(self, request: ClearImageCacheRequest) -> Result<()> { + self.state + .clear_image_cache(&request.image_hash, &request.config_hash) + } +} + +impl RpcCall<KmsState> for AdminRpcHandler { + type PrpcService = AdminServer<Self>; + + fn construct(context: CallContext<'_, KmsState>) -> Result<Self> { + Ok(AdminRpcHandler { + state: context.state.clone(), + }) + } +} diff --git a/dstack/kms/src/config.rs b/dstack/kms/src/config.rs index ce66654c0..ba3f9f93e 100644 --- a/dstack/kms/src/config.rs +++ b/dstack/kms/src/config.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 +use dstack_attest::attestation::AttestationVerifierConfig; use load_config::load_config; use rocket::figment::Figment; use serde::Deserialize; @@ -34,15 +35,8 @@ pub(crate) struct ImageConfig { #[derive(Debug, Clone, Deserialize)] pub(crate) struct KmsConfig { pub cert_dir: PathBuf, - pub pccs_url: Option<String>, - /// Optional AMD KDS-compatible base URL used for SEV-SNP collateral requests. - /// - /// Empty by default. When set, the KMS process exports this base URL for - /// dstack-attest before any attestation verification happens. The base URL - /// must expose AMD KDS-compatible paths under `/vcek/v1`, e.g. - /// `https://kdsintf.amd.com/vcek/v1` or a trusted mirror/cache. #[serde(default)] - pub amd_kds_base_url: Option<String>, + pub attestation: AttestationVerifierConfig, pub auth_api: AuthApi, pub onboard: OnboardConfig, pub image: ImageConfig, @@ -59,10 +53,18 @@ pub(crate) struct KmsConfig { /// mirroring `sev_snp_key_release`. #[serde(default)] pub aws_nitro_tpm_key_release: bool, - #[serde(with = "serde_human_bytes")] - pub admin_token_hash: Vec<u8>, #[serde(default)] pub site_name: String, + /// Whether the KMS embeds an attestation in its own RPC certificate. + /// Defaults to `true`. Set `false` only for local dev/testing where the KMS + /// runs outside a TEE and cannot reach a guest agent socket. + /// + /// This narrows what the KMS asserts about itself; it relaxes no + /// verification anywhere. Quotes presented *to* the KMS are still fully + /// checked, so key release stays gated, and another KMS refuses to onboard + /// from an unattested one. + #[serde(default = "default_true")] + pub attest_rpc_cert: bool, /// Whether trusted RPCs require the KMS to first attest itself to its /// own auth API. Defaults to `true` (strict). Set `false` only for local /// dev/testing where the KMS runs outside a TEE and cannot reach a guest @@ -70,6 +72,10 @@ pub(crate) struct KmsConfig { #[serde(default = "default_true")] pub enforce_self_authorization: bool, pub metrics: MetricsConfig, + /// Admin API listener + authentication. The admin RPCs (e.g. + /// `ClearImageCache`) are served here, behind the shared HTTP authenticator. + #[serde(default)] + pub admin: AdminConfig, } #[derive(Debug, Clone, Deserialize)] @@ -78,6 +84,29 @@ pub(crate) struct MetricsConfig { pub enabled: bool, } +/// Admin API listener + authentication, mirroring the gateway `[core.admin]` +/// section. The listen `address`/`port` are read from the same `[core.admin]` +/// section by Rocket. The token travels in the `Authorization`/`X-Admin-Token` +/// header. +#[derive(Debug, Clone, Default, Deserialize)] +pub(crate) struct AdminConfig { + /// Whether to serve the admin API at all. + #[serde(default)] + pub enabled: bool, + /// Shared admin token required to call any admin RPC. Can also be supplied + /// via `DSTACK_KMS_ADMIN_TOKEN` / `ADMIN_API_TOKEN`. Required unless + /// `insecure_no_auth = true`. + #[serde(default)] + pub auth_token: String, + /// Optional Apache bcrypt htpasswd file, accepted in addition to the token. + #[serde(default)] + pub htpasswd_file: PathBuf, + /// Development-only escape hatch: serve the admin API with no auth. Never + /// enable on a network-reachable listener. + #[serde(default)] + pub insecure_no_auth: bool, +} + fn default_true() -> bool { true } @@ -160,3 +189,52 @@ pub(crate) struct OnboardConfig { pub enabled: bool, pub auto_bootstrap_domain: String, } + +#[cfg(test)] +mod tests { + use super::*; + use rocket::figment::providers::{Format, Toml}; + + #[test] + fn default_config_parses_with_admin_disabled_and_no_hash() { + let figment = load_config_figment(None); + let config: KmsConfig = figment + .focus("core") + .extract() + .expect("kms.toml must parse into KmsConfig"); + assert!(!config.admin.enabled, "admin must be off by default"); + assert!( + config.admin.auth_token.is_empty(), + "default admin token must be empty (fail-closed)" + ); + assert!(!config.admin.insecure_no_auth); + } + + #[test] + fn rpc_cert_is_attested_by_default() { + let figment = load_config_figment(None); + let config: KmsConfig = figment + .focus("core") + .extract() + .expect("kms.toml must parse into KmsConfig"); + assert!( + config.attest_rpc_cert, + "the KMS must attest its own RPC certificate unless explicitly told not to" + ); + } + + #[test] + fn omitting_the_key_keeps_the_attested_default() { + // Configs written before this key existed must keep working, and must + // land on the attested side. + #[derive(Deserialize)] + struct Probe { + #[serde(default = "default_true")] + attest_rpc_cert: bool, + } + let probe: Probe = Figment::from(Toml::string("")) + .extract() + .expect("an absent attest_rpc_cert must parse"); + assert!(probe.attest_rpc_cert); + } +} diff --git a/dstack/kms/src/crypto.rs b/dstack/kms/src/crypto.rs index 1037e8066..faa8e01f8 100644 --- a/dstack/kms/src/crypto.rs +++ b/dstack/kms/src/crypto.rs @@ -4,8 +4,8 @@ use anyhow::{Context, Result}; use k256::ecdsa::SigningKey; -use sha3::{Digest, Keccak256}; +use ra_tls::api_v1::sign_recoverable_keccak256; use ra_tls::kdf; pub(crate) fn derive_k256_key( @@ -35,11 +35,9 @@ pub(crate) fn sign_message( appid: &[u8], message: &[u8], ) -> Result<Vec<u8>> { - let digest = Keccak256::new_with_prefix([prefix, b":", appid, message].concat()); - let (signature, recid) = key.sign_digest_recoverable(digest)?; - let mut signature_bytes = signature.to_vec(); - signature_bytes.push(recid.to_byte()); - Ok(signature_bytes) + // Same 65-byte `r || s || v` envelope every dstack chain link uses; the + // preimage is what differs between them. + sign_recoverable_keccak256(key, &[prefix, b":", appid, message].concat()) } /// Sign a message with a timestamp to prevent replay attacks. @@ -52,10 +50,152 @@ pub(crate) fn sign_message_with_timestamp( message: &[u8], ) -> Result<Vec<u8>> { let timestamp_bytes = timestamp.to_be_bytes(); - let digest = - Keccak256::new_with_prefix([prefix, b":", appid, ×tamp_bytes[..], message].concat()); - let (signature, recid) = key.sign_digest_recoverable(digest)?; - let mut signature_bytes = signature.to_vec(); - signature_bytes.push(recid.to_byte()); - Ok(signature_bytes) + sign_recoverable_keccak256( + key, + &[prefix, b":", appid, ×tamp_bytes[..], message].concat(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use sha3::{Digest as _, Keccak256}; + + /// The pre-refactor envelope, reproduced verbatim from the implementation + /// that shipped before `sign_message` moved onto the shared helper. + /// + /// Kept as executable code rather than prose so the compatibility claim is + /// checked on every run: whatever KMS sends to already-deployed CVMs was + /// produced by exactly this, and a divergence here is a divergence in + /// issued app keys and their signatures. + fn legacy_sign_message( + key: &SigningKey, + prefix: &[u8], + appid: &[u8], + message: &[u8], + ) -> Vec<u8> { + let digest = Keccak256::new_with_prefix([prefix, b":", appid, message].concat()); + let (signature, recid) = key.sign_digest_recoverable(digest).unwrap(); + let mut signature_bytes = signature.to_vec(); + signature_bytes.push(recid.to_byte()); + signature_bytes + } + + /// The pre-refactor timestamped envelope, likewise verbatim. + fn legacy_sign_message_with_timestamp( + key: &SigningKey, + prefix: &[u8], + appid: &[u8], + timestamp: u64, + message: &[u8], + ) -> Vec<u8> { + let timestamp_bytes = timestamp.to_be_bytes(); + let digest = Keccak256::new_with_prefix( + [prefix, b":", appid, ×tamp_bytes[..], message].concat(), + ); + let (signature, recid) = key.sign_digest_recoverable(digest).unwrap(); + let mut signature_bytes = signature.to_vec(); + signature_bytes.push(recid.to_byte()); + signature_bytes + } + + /// Moving `sign_message` onto `ra_tls::api_v1::sign_recoverable_keccak256` + /// must not change one byte KMS sends to an already-deployed CVM. + /// + /// ECDSA here is deterministic (RFC 6979), so this is an exact comparison + /// rather than a signature check. The cases below are the two shapes KMS + /// actually signs; broader coverage of the KMS surface belongs to the + /// test-infrastructure effort in PR #841, not here. + #[test] + fn the_shared_envelope_matches_the_pre_refactor_bytes() { + let key = SigningKey::from_slice(&[7_u8; 32]).unwrap(); + let cases: [(&[u8], &[u8], &[u8]); 4] = [ + (b"dstack-kms-issued", b"app-a", &[0x42_u8; 33]), + (b"dstack-kms-issued", b"", &[]), + (b"dstack-env-encrypt-pubkey", b"app-b", &[0x01_u8; 32]), + (b"x", b"\x00\xff", &[0xde, 0xad, 0xbe, 0xef]), + ]; + for (prefix, appid, message) in cases { + assert_eq!( + sign_message(&key, prefix, appid, message).unwrap(), + legacy_sign_message(&key, prefix, appid, message), + "envelope changed for prefix {prefix:?}" + ); + } + + for timestamp in [0_u64, 1, 1_786_194_000, u64::MAX] { + assert_eq!( + sign_message_with_timestamp(&key, b"p", b"app", timestamp, b"msg").unwrap(), + legacy_sign_message_with_timestamp(&key, b"p", b"app", timestamp, b"msg"), + "timestamped envelope changed at {timestamp}" + ); + } + } + + /// Golden bytes, so the reference implementation above cannot drift along + /// with the real one and hide a change. + /// + /// Captured from the pre-refactor code. Do not update these to match new + /// output -- a mismatch means KMS would issue different key signatures than + /// the ones deployed CVMs already hold. + #[test] + fn issued_app_key_signatures_match_their_golden_vectors() { + let root = SigningKey::from_slice(&[7_u8; 32]).unwrap(); + let (derived, signature) = derive_k256_key(&root, b"app-a").unwrap(); + + assert_eq!( + hex::encode(derived.to_bytes()), + "ed0fd39ce7c26a185f396945168972807b2f77820f14ee4bda9e1f46e8b4596d", + "the issued app key changed" + ); + assert_eq!( + hex::encode(&signature), + "e12a5f4567f6d9f80b01944dea333bbed5b418347f4260ed95e9ee198a03014e1716a40d2ed6cd367e335b33e231510aa98216e62b631f2b75c181c2f989656800", + "the issued app key signature changed" + ); + // ...and it is still what the pre-refactor envelope produces. + assert_eq!( + signature, + legacy_sign_message( + &root, + b"dstack-kms-issued", + b"app-a", + &derived.verifying_key().to_sec1_bytes() + ) + ); + } + + #[test] + fn environment_public_key_signatures_bind_domain_app_key_and_timestamp() { + let key = SigningKey::from_slice(&[7_u8; 32]).unwrap(); + let prefix = b"dstack-env-encrypt-pubkey"; + let app_id = b"app-a"; + let timestamp = 1_786_194_000; + let public_key = [0x42_u8; 32]; + let baseline = + sign_message_with_timestamp(&key, prefix, app_id, timestamp, &public_key).unwrap(); + + assert_eq!(baseline.len(), 65); + assert_eq!( + baseline, + sign_message_with_timestamp(&key, prefix, app_id, timestamp, &public_key).unwrap() + ); + assert_ne!( + baseline, + sign_message_with_timestamp(&key, b"other-domain", app_id, timestamp, &public_key) + .unwrap() + ); + assert_ne!( + baseline, + sign_message_with_timestamp(&key, prefix, b"app-b", timestamp, &public_key).unwrap() + ); + assert_ne!( + baseline, + sign_message_with_timestamp(&key, prefix, app_id, timestamp + 1, &public_key).unwrap() + ); + assert_ne!( + baseline, + sign_message_with_timestamp(&key, prefix, app_id, timestamp, &[0x43_u8; 32]).unwrap() + ); + } } diff --git a/dstack/kms/src/ct_log.rs b/dstack/kms/src/ct_log.rs deleted file mode 100644 index de5c4b6bd..000000000 --- a/dstack/kms/src/ct_log.rs +++ /dev/null @@ -1,92 +0,0 @@ -// SPDX-FileCopyrightText: © 2024 Phala Network <dstack@phala.network> -// -// SPDX-License-Identifier: Apache-2.0 - -use std::path::{Path, PathBuf}; - -use anyhow::{Context, Result}; -use chrono::Utc; -use fs_err as fs; - -#[allow(dead_code)] -pub(crate) fn iter_ct_log_files(log_dir: &Path) -> Result<impl Iterator<Item = PathBuf>> { - // Certs files at log_dir/YYYYMMDD/xxx.cert - let day_dirs = fs::read_dir(log_dir)?.filter_map(|entry| { - let path = entry.ok()?.path(); - if path.is_dir() { - Some(path) - } else { - None - } - }); - - Ok(day_dirs - .flat_map(|dir| { - let iter = fs::read_dir(dir).ok()?.filter_map(|entry| { - let path = entry.ok()?.path(); - if path.is_file() && path.ends_with(".cert") { - Some(path) - } else { - None - } - }); - Some(iter) - }) - .flatten()) -} - -pub(crate) fn ct_log_write_cert(app_id: &str, cert: &str, log_dir: &str) -> Result<()> { - // filename: %Y%d%m-%H%M%S.%sn.%app_id.cert - let log_dir = Path::new(log_dir); - let now = Utc::now(); - let day = now.format("%Y%d%m").to_string(); - let base_filename = format!("{}-{app_id}", now.format("%Y%d%m-%H%M%S")); - let day_dir = log_dir.join(day); - fs::create_dir_all(&day_dir).context("failed to create ct log dir")?; - let cert_log_path = find_available_filename(&day_dir, &base_filename) - .context("failed to find available filename")?; - fs::write(cert_log_path, cert).context("faile to write ct log cert")?; - Ok(()) -} - -fn binary_search(mut upper: usize, is_ok: impl Fn(usize) -> bool) -> Option<usize> { - let mut lower = 0; - if is_ok(0) { - return Some(0); - } - if !is_ok(upper) { - return None; - } - while lower < upper { - let mid = lower + (upper - lower) / 2; - if is_ok(mid) { - upper = mid; - } else { - lower = mid + 1; - } - } - Some(upper) -} - -fn find_available_filename(dir: &Path, base_filename: &str) -> Option<PathBuf> { - fn mk_filename(dir: &Path, base_filename: &str, index: usize) -> PathBuf { - dir.join(format!("{base_filename}.{index}.cert")) - } - let available_index = binary_search(4096, |i| { - !std::path::Path::new(&mk_filename(dir, base_filename, i)).exists() - })?; - Some(mk_filename(dir, base_filename, available_index)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_binary_search_simple() { - for i in 0..=4096 { - let result = binary_search(4096, |x| x >= i); - assert_eq!(result, Some(i)); - } - } -} diff --git a/dstack/kms/src/main.rs b/dstack/kms/src/main.rs index 4bc063f7a..b6dae8f59 100644 --- a/dstack/kms/src/main.rs +++ b/dstack/kms/src/main.rs @@ -6,29 +6,27 @@ use anyhow::{anyhow, bail, Context, Result}; use clap::Parser; use config::KmsConfig; use main_service::{KmsState, RpcHandler}; +use ra_rpc::ratls_client_verifier::RaTlsClientAuth; use ra_rpc::rocket_helper::QuoteVerifier; +use ra_tls::attestation::AttestationVerifier; use rocket::{ fairing::AdHoc, figment::{providers::Serialized, Figment}, response::content::{RawHtml, RawText}, + tls::Resolver as _, Shutdown, State, }; use tracing::{info, warn}; +mod admin_auth; +mod admin_service; mod config; -// mod ct_log; mod crypto; mod main_service; mod onboard_service; fn app_version() -> String { - const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION"); - const VERSION: &str = git_version::git_version!( - args = ["--abbrev=20", "--always", "--dirty=-modified"], - prefix = "git:", - fallback = "unknown" - ); - format!("v{CARGO_PKG_VERSION} ({VERSION})") + dstack_build_info::app_version!() } #[derive(Parser)] @@ -53,30 +51,42 @@ async fn run_onboard_service(kms_config: KmsConfig, figment: Figment) -> Result< } if !kms_config.onboard.auto_bootstrap_domain.is_empty() { - onboard_service::bootstrap_keys(&kms_config).await?; + let verifier = AttestationVerifier::load(&kms_config.attestation) + .context("failed to load attestation verifier")?; + onboard_service::bootstrap_keys(&kms_config, &verifier).await?; return Ok(()); } - let state = OnboardState::new(kms_config); + let state = OnboardState::new(kms_config)?; let figment = figment .clone() .merge(Serialized::defaults(figment.find_value("core.onboard")?)); // Remove section tls - let _ = rocket::custom(figment) - .mount("/", rocket::routes![index, finish]) + let rocket = rocket::custom(figment) + .mount("/", rocket::routes![index, finish, health]) .mount( "/prpc", ra_rpc::prpc_routes!(OnboardState, OnboardHandler, trim: "Onboard."), ) - .manage(state) + .manage(state.clone()) + .ignite() + .await + .map_err(|err| anyhow!(err.to_string()))?; + state.set_shutdown(rocket.shutdown())?; + let _ = rocket .launch() .await .map_err(|err| anyhow!(err.to_string()))?; Ok(()) } +#[rocket::get("/health")] +fn health() -> RawText<&'static str> { + RawText("OK") +} + #[rocket::get("/metrics")] fn metrics(state: &State<KmsState>) -> RawText<String> { RawText(state.metrics().render_prometheus()) @@ -105,19 +115,6 @@ fn record_attestation_metrics(req: &rocket::Request<'_>, res: &rocket::Response< .record_attestation_request(res.status().code >= 400); } -fn configure_amd_kds_base_from_config(config: &KmsConfig) { - let Some(base_url) = config - .amd_kds_base_url - .as_deref() - .map(str::trim) - .filter(|base_url| !base_url.is_empty()) - else { - return; - }; - std::env::set_var("DSTACK_AMD_KDS_BASE_URL", base_url); - info!("AMD SEV-SNP KDS base URL configured"); -} - #[rocket::main] async fn main() -> Result<()> { { @@ -129,7 +126,14 @@ async fn main() -> Result<()> { let figment = config::load_config_figment(args.config.as_deref()); let config: KmsConfig = figment.focus("core").extract()?; - configure_amd_kds_base_from_config(&config); + + if !config.attest_rpc_cert { + warn!( + "attest_rpc_cert = false; the KMS RPC certificate carries no attestation, so \ + guests cannot verify which KMS they are talking to. Intended for local \ + development only" + ); + } if config.onboard.enabled && !config.keys_exists() { info!("Onboarding"); @@ -141,6 +145,12 @@ async fn main() -> Result<()> { info!("Updating certs"); if let Err(err) = onboard_service::update_certs(&config).await { + if config.attest_rpc_cert { + return Err(err).context( + "Failed to reissue the attested KMS RPC certificate; refusing to start with a \ + potentially unattested certificate", + ); + } warn!("Failed to update certs: {err}"); }; @@ -150,14 +160,33 @@ async fn main() -> Result<()> { info!(" /prpc/{method}"); } - let pccs_url = config.pccs_url.clone(); - let amd_kds_base_url = config.amd_kds_base_url.clone(); let metrics_enabled = config.metrics.enabled; + let admin_config = config.admin.clone(); + // build the admin listener figment from `[core.admin]` before `config` is + // moved into the state; the fairing is built now so a misconfigured admin + // (enabled but no credential) fails fast before we start serving. + let admin_setup = if admin_config.enabled { + let admin_value = figment + .find_value("core.admin") + .context("core.admin section not found")?; + let admin_figment = + Figment::from(rocket::Config::default()).merge(Serialized::defaults(admin_value)); + let admin_fairing = admin_auth::AdminAuthFairing::from_config(&admin_config)?; + Some((admin_figment, admin_fairing)) + } else { + None + }; let state = main_service::KmsState::new(config).context("Failed to initialize KMS state")?; + let quote_verifier = QuoteVerifier::new(state.attestation_verifier()); let figment = figment .clone() .merge(Serialized::defaults(figment.find_value("rpc")?)); let mut rocket = rocket::custom(figment) + // Verify client certificates by their attestation rather than by issuer. The + // certificates guests and onboarding mint from the temp CA today keep working + // unchanged - they are now accepted for the attestation they carry rather than + // for who signed them - and a self-issued certificate would be accepted too. + .attach(RaTlsClientAuth::fairing()) .attach(AdHoc::on_response("Add app version header", |_req, res| { Box::pin(async move { res.set_raw_header("X-App-Version", app_version()); @@ -167,7 +196,8 @@ async fn main() -> Result<()> { "/prpc", ra_rpc::prpc_routes!(KmsState, RpcHandler, trim: "KMS."), ) - .manage(state); + .mount("/", rocket::routes![health]) + .manage(state.clone()); if metrics_enabled { info!("Prometheus metrics endpoint enabled at /metrics"); @@ -179,12 +209,35 @@ async fn main() -> Result<()> { .mount("/", rocket::routes![metrics]); } - let verifier = QuoteVerifier::new_with_amd_kds_base(pccs_url, amd_kds_base_url); - rocket = rocket.manage(verifier); - - rocket - .launch() - .await - .map_err(|err| anyhow!(err.to_string()))?; + rocket = rocket.manage(quote_verifier); + + let main_srv = rocket.launch(); + match admin_setup { + Some((admin_figment, admin_fairing)) => { + if admin_config.insecure_no_auth { + warn!( + "admin API is served with insecure_no_auth = true; the admin RPCs are exposed without authentication" + ); + } else { + info!("admin API authentication enabled"); + } + let admin_srv = rocket::custom(admin_figment) + .attach(admin_fairing) + .mount("/", admin_auth::routes()) + .mount( + "/prpc", + ra_rpc::prpc_routes!(KmsState, admin_service::AdminRpcHandler, trim: "Admin."), + ) + .manage(state) + .launch(); + tokio::try_join!( + async { main_srv.await.map_err(|err| anyhow!(err.to_string())) }, + async { admin_srv.await.map_err(|err| anyhow!(err.to_string())) }, + )?; + } + None => { + main_srv.await.map_err(|err| anyhow!(err.to_string()))?; + } + } Ok(()) } diff --git a/dstack/kms/src/main_service.rs b/dstack/kms/src/main_service.rs index 6e3a4694d..726a0a341 100644 --- a/dstack/kms/src/main_service.rs +++ b/dstack/kms/src/main_service.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use std::{ - path::{Path, PathBuf}, + path::Path, sync::{ atomic::{AtomicU64, Ordering}, Arc, @@ -13,21 +13,20 @@ use std::{ use anyhow::{bail, Context, Result}; use dstack_kms_rpc::{ kms_server::{KmsRpc, KmsServer}, - AppId, AppKeyResponse, ClearImageCacheRequest, GetAppKeyRequest, GetKmsKeyRequest, - GetMetaResponse, GetTempCaCertResponse, KmsKeyResponse, KmsKeys, PublicKeyResponse, - SignCertRequest, SignCertResponse, + AppId, AppKeyResponse, GetAppKeyRequest, GetKmsKeyRequest, GetMetaResponse, + GetTempCaCertResponse, KmsKeyResponse, KmsKeys, PublicKeyResponse, SignCertRequest, + SignCertResponse, }; use dstack_verifier::{CvmVerifier, VerificationDetails}; use fs_err as fs; use k256::ecdsa::SigningKey; use ra_rpc::{CallContext, RpcCall}; use ra_tls::{ - attestation::{AttestationMode, VerifiedAttestation}, + attestation::{AttestationVerifier, TeeVariant, VerifiedAttestation}, cert::{CaCert, CertRequest, CertSigningRequestV1, CertSigningRequestV2, Csr}, kdf, }; use scale::Decode; -use sha2::Digest; use tokio::sync::OnceCell; use tracing::{info, warn}; use upgrade_authority::{build_boot_info, ensure_app_id_len, local_kms_boot_info, BootInfo}; @@ -60,6 +59,7 @@ pub struct KmsStateInner { temp_ca_cert: String, temp_ca_key: String, verifier: CvmVerifier, + attestation_verifier: Arc<AttestationVerifier>, self_boot_info: OnceCell<BootInfo>, metrics: KmsMetrics, } @@ -95,7 +95,44 @@ impl KmsMetrics { } } +/// remove a single cache entry (a hex-named subdir/file) under `parent_dir`, or +/// everything when `sub_dir == "all"`. A non-hex key is rejected to keep the +/// deletion confined to the cache. +fn remove_cache(parent_dir: &Path, sub_dir: &str) -> Result<()> { + if sub_dir.is_empty() { + return Ok(()); + } + if sub_dir == "all" { + if parent_dir.exists() { + fs::remove_dir_all(parent_dir)?; + } + return Ok(()); + } + if !sub_dir.chars().all(|c| c.is_ascii_hexdigit()) { + bail!("invalid cache key"); + } + let path = parent_dir.join(sub_dir); + if path.is_dir() { + fs::remove_dir_all(path)?; + } else if path.is_file() { + fs::remove_file(path)?; + } + Ok(()) +} + impl KmsState { + /// clear cached image and measurement material for the given hashes. Used by + /// the admin `ClearImageCache` RPC; authorization is enforced by the admin + /// listener's HTTP authenticator, not here. + pub(crate) fn clear_image_cache(&self, image_hash: &str, config_hash: &str) -> Result<()> { + let images_dir = self.config.image.cache_dir.join("images"); + remove_cache(&images_dir, image_hash).context("failed to clear image cache")?; + // measurement cache is kept by the verifier under measurements/. + let mr_cache_dir = self.config.image.cache_dir.join("measurements"); + remove_cache(&mr_cache_dir, config_hash).context("failed to clear measurement cache")?; + Ok(()) + } + pub fn new(config: KmsConfig) -> Result<Self> { let root_ca = CaCert::load(config.root_ca_cert(), config.root_ca_key()) .context("Failed to load root CA certificate")?; @@ -106,11 +143,15 @@ impl KmsState { fs::read_to_string(config.tmp_ca_key()).context("Faeild to read temp ca key")?; let temp_ca_cert = fs::read_to_string(config.tmp_ca_cert()).context("Faeild to read temp ca cert")?; + let attestation_verifier = Arc::new( + AttestationVerifier::load(&config.attestation) + .context("failed to load attestation verifier")?, + ); let verifier = CvmVerifier::new( config.image.cache_dir.display().to_string(), config.image.download_url.clone(), config.image.download_timeout, - config.pccs_url.clone(), + attestation_verifier.clone(), ); if !config.enforce_self_authorization { warn!( @@ -125,6 +166,7 @@ impl KmsState { temp_ca_cert, temp_ca_key, verifier, + attestation_verifier, self_boot_info: OnceCell::new(), metrics: KmsMetrics::default(), }), @@ -134,6 +176,10 @@ impl KmsState { pub(crate) fn metrics(&self) -> &KmsMetrics { &self.inner.metrics } + + pub(crate) fn attestation_verifier(&self) -> Arc<AttestationVerifier> { + self.inner.attestation_verifier.clone() + } } pub struct RpcHandler { @@ -170,11 +216,11 @@ fn ensure_key_release_allowed( snp_enabled: bool, aws_nitro_tpm_enabled: bool, ) -> Result<()> { - match boot_info.attestation_mode { - AttestationMode::DstackAmdSevSnp if !snp_enabled => { + match boot_info.tee_variant { + TeeVariant::DstackAmdSevSnp if !snp_enabled => { bail!("amd sev-snp key release is not enabled") } - AttestationMode::DstackAwsNitroTpm if !aws_nitro_tpm_enabled => { + TeeVariant::DstackAwsNitroTpm if !aws_nitro_tpm_enabled => { bail!("aws nitro-tpm key release is not enabled") } _ => Ok(()), @@ -200,7 +246,7 @@ impl RpcHandler { let boot_info = self .state .self_boot_info - .get_or_try_init(|| local_kms_boot_info(self.state.config.pccs_url.as_deref())) + .get_or_try_init(|| local_kms_boot_info(&self.state.attestation_verifier)) .await .context("Failed to load cached self boot info")?; let response = self @@ -236,43 +282,6 @@ impl RpcHandler { .await } - fn image_cache_dir(&self) -> PathBuf { - self.state.config.image.cache_dir.join("images") - } - - fn remove_cache(&self, parent_dir: &Path, sub_dir: &str) -> Result<()> { - if sub_dir.is_empty() { - return Ok(()); - } - - if sub_dir == "all" { - fs::remove_dir_all(parent_dir)?; - return Ok(()); - } - - if !sub_dir.chars().all(|c| c.is_ascii_hexdigit()) { - bail!("Invalid cache key"); - } - - let path = parent_dir.join(sub_dir); - - if path.is_dir() { - fs::remove_dir_all(path)?; - } else if path.is_file() { - fs::remove_file(path)?; - } - - Ok(()) - } - - fn ensure_admin(&self, token: &str) -> Result<()> { - let token_hash = sha2::Sha256::new_with_prefix(token).finalize(); - if token_hash.as_slice() != self.state.config.admin_token_hash.as_slice() { - bail!("Invalid token"); - } - Ok(()) - } - async fn verify_os_image_hash( &self, vm_config: String, @@ -311,7 +320,7 @@ impl RpcHandler { // SNP rootfs/app/config binding is handled by the SNP launch-measurement // helper above. The legacy OS-image verifier is TDX-oriented and still // rejects SNP quotes; keep SNP on the explicit fail-closed helper path. - if boot_info.attestation_mode != AttestationMode::DstackAmdSevSnp { + if boot_info.tee_variant != TeeVariant::DstackAmdSevSnp { self.verify_os_image_hash(vm_config_str.into(), att) .await .context("Failed to verify os image hash")?; @@ -485,6 +494,20 @@ impl KmsRpc for RpcHandler { }) } + /// Serve the temp CA certificate and key. + /// + /// Both current callers - guests at boot, and KMS-to-KMS onboarding + /// ([`crate::onboard_service`]) - fetch this CA and mint their client certificate + /// from it, because the KMS used to pin the CA for mutual TLS and a self-issued + /// certificate had nothing to chain to. + /// + /// That pin is gone: client certificates are now verified by the attestation they + /// carry (`ra_rpc::ratls_client_verifier`), so a self-issued certificate would be + /// accepted on the same terms. Neither caller has been migrated yet, so this RPC + /// still has to work. + /// + /// The key it returns authenticates nobody: it is handed to any caller. Removing + /// this RPC needs both callers migrated first. async fn get_temp_ca_cert(self) -> Result<GetTempCaCertResponse> { let self_boot_info = self .ensure_self_allowed() @@ -527,7 +550,7 @@ impl KmsRpc for RpcHandler { .attestation .clone() .into_v1() - .verify_with_ra_pubkey(&csr.pubkey, self.state.config.pccs_url.as_deref()) + .verify_with_ra_pubkey(&csr.pubkey, &self.state.attestation_verifier) .await .context("Quote verification failed")?; let app_info = self @@ -550,17 +573,6 @@ impl KmsRpc for RpcHandler { ], }) } - - async fn clear_image_cache(self, request: ClearImageCacheRequest) -> Result<()> { - self.ensure_admin(&request.token)?; - self.remove_cache(&self.image_cache_dir(), &request.image_hash) - .context("Failed to clear image cache")?; - // Clear measurement cache (now handled by verifier's cache in measurements/ dir) - let mr_cache_dir = self.state.config.image.cache_dir.join("measurements"); - self.remove_cache(&mr_cache_dir, &request.config_hash) - .context("Failed to clear measurement cache")?; - Ok(()) - } } impl RpcCall<KmsState> for RpcHandler { @@ -585,7 +597,30 @@ mod tests { compute_expected_measurement, MeasurementInput, OvmfSectionParam, }; use cc_eventlog::RuntimeEvent; - use sha2::{Sha256, Sha384}; + use sha2::{Digest, Sha256, Sha384}; + + #[test] + fn remove_cache_only_deletes_the_named_hex_entry() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let keep = root.join("cafe"); + let drop = root.join("beef"); + fs::create_dir_all(&keep).unwrap(); + fs::create_dir_all(&drop).unwrap(); + + remove_cache(root, "beef").unwrap(); + assert!(!drop.exists(), "named entry must be removed"); + assert!(keep.exists(), "other entries must be kept"); + + // empty key is a no-op; a non-hex key is rejected before touching disk. + remove_cache(root, "").unwrap(); + assert!(remove_cache(root, "../escape").is_err()); + assert!(keep.exists()); + + // "all" clears the whole cache dir. + remove_cache(root, "all").unwrap(); + assert!(!root.exists()); + } use std::collections::BTreeMap; fn hex_of(byte: u8, len: usize) -> String { @@ -633,6 +668,7 @@ mod tests { dstack_types::mr_config::MrConfigV3::new( vec![0x11; 20], vec![0x22; 32], + None, dstack_types::KeyProviderKind::None, Vec::new(), vec![0x99; 20], @@ -759,7 +795,7 @@ mod tests { } fn runtime_event(event: &str, payload: Vec<u8>) -> RuntimeEvent { - RuntimeEvent::new(event.to_string(), payload) + RuntimeEvent::new(event.to_string(), payload, Default::default()) } fn verified_aws_nitro_tpm_attestation( @@ -853,10 +889,7 @@ mod tests { .expect("aws nitrotpm attestation should produce KMS boot info"); let pcrs = verified_aws_nitro_tpm_pcrs(&attestation); - assert_eq!( - boot_info.attestation_mode, - AttestationMode::DstackAwsNitroTpm - ); + assert_eq!(boot_info.tee_variant, TeeVariant::DstackAwsNitroTpm); assert_eq!(boot_info.tcb_status, "UpToDate"); assert!(boot_info.advisory_ids.is_empty()); assert_eq!(boot_info.app_id, vec![0x11; 20]); @@ -911,7 +944,7 @@ mod tests { ensure_key_release_allowed(&boot_info, false, true).unwrap(); // A TDX boot info is unaffected by the AWS gate even when it is disabled. - boot_info.attestation_mode = AttestationMode::DstackTdx; + boot_info.tee_variant = TeeVariant::DstackTdx; ensure_key_release_allowed(&boot_info, false, false).unwrap(); } @@ -1004,7 +1037,7 @@ mod tests { let boot_info = build_boot_info_for_attestation(&attestation, false, &vm_config) .expect("snp attestation should build boot info through vm_config path"); - assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); + assert_eq!(boot_info.tee_variant, TeeVariant::DstackAmdSevSnp); assert_eq!(boot_info.mr_aggregated.len(), 32); assert_eq!(boot_info.device_id, vec![0xab; 64]); assert_eq!(boot_info.app_id, vec![0x11; 20]); @@ -1026,7 +1059,7 @@ mod tests { let boot_info = build_boot_info_for_attestation(&attestation, false, "") .expect("snp local KMS attestation should use embedded vm_config"); - assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); + assert_eq!(boot_info.tee_variant, TeeVariant::DstackAmdSevSnp); assert_eq!(boot_info.mr_aggregated.len(), 32); assert_eq!(boot_info.app_id, vec![0x11; 20]); } @@ -1041,7 +1074,7 @@ mod tests { let boot_info = build_boot_info_for_attestation(&attestation, false, &vm_config) .expect("self-contained SNP vm_config should not require KMS-local sev_snp config"); - assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); + assert_eq!(boot_info.tee_variant, TeeVariant::DstackAmdSevSnp); assert_eq!(boot_info.device_id, vec![0xab; 64]); } diff --git a/dstack/kms/src/main_service/amd_attest.rs b/dstack/kms/src/main_service/amd_attest.rs index bcbbdbf4d..a9a943131 100644 --- a/dstack/kms/src/main_service/amd_attest.rs +++ b/dstack/kms/src/main_service/amd_attest.rs @@ -25,7 +25,7 @@ use anyhow::{bail, Context, Result}; use dstack_types::{mr_config::MrConfigV3, KeyProviderInfo}; -use ra_tls::attestation::{AttestationMode, VerifiedAttestation}; +use ra_tls::attestation::{TeeVariant, VerifiedAttestation}; use sha2::{Digest, Sha256}; use super::upgrade_authority::BootInfo; @@ -109,13 +109,13 @@ fn build_amd_snp_boot_info_with_tcb_status( let key_provider_info = mr_config_key_provider_info(&mr_config)?; Ok(BootInfo { - attestation_mode: AttestationMode::DstackAmdSevSnp, + tee_variant: TeeVariant::DstackAmdSevSnp, mr_aggregated, os_image_hash: os_image_hash.to_vec(), mr_system, - app_id: mr_config.app_id.clone(), + app_id: mr_config.app_id.clone().unwrap_or_default(), compose_hash: mr_config.compose_hash.clone(), - instance_id: mr_config.instance_id.clone(), + instance_id: mr_config.instance_id.clone().unwrap_or_default(), device_id: verified_chip_id.to_vec(), key_provider_info, tcb_status: tcb_status.to_string(), @@ -182,7 +182,7 @@ fn parse_measurement_input_from_vm_config(vm_config: &str) -> Result<Measurement fn mr_config_key_provider_info(mr_config: &MrConfigV3) -> Result<Vec<u8>> { serde_json::to_vec(&KeyProviderInfo::new( mr_config.key_provider_name().to_string(), - hex::encode(&mr_config.key_provider_id), + hex::encode(mr_config.key_provider_id.as_deref().unwrap_or_default()), )) .context("failed to serialize key provider info") } @@ -193,6 +193,7 @@ fn test_mr_config(app_id: Vec<u8>, compose_hash: Vec<u8>) -> MrConfigV3 { MrConfigV3::new( app_id, compose_hash, + None, dstack_types::KeyProviderKind::None, Vec::new(), instance_id, @@ -365,7 +366,7 @@ mod tests { let boot_info = build_amd_snp_boot_info(&verified, &chip_id, &input) .expect("matching measurement should build snp boot info"); - assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); + assert_eq!(boot_info.tee_variant, TeeVariant::DstackAmdSevSnp); assert_eq!(boot_info.mr_aggregated.len(), 32); assert_eq!(boot_info.device_id, chip_id.to_vec()); assert_eq!(boot_info.app_id, vec![0x11; 20]); diff --git a/dstack/kms/src/main_service/upgrade_authority.rs b/dstack/kms/src/main_service/upgrade_authority.rs index a577849ac..8b1c20742 100644 --- a/dstack/kms/src/main_service/upgrade_authority.rs +++ b/dstack/kms/src/main_service/upgrade_authority.rs @@ -5,12 +5,11 @@ use super::build_boot_info_for_attestation; use crate::config::{AuthApi, KmsConfig}; use anyhow::{bail, Context, Result}; -use dstack_guest_agent_rpc::{ +use dstack_guest_agent_rpc::v0::{ dstack_guest_client::DstackGuestClient, AttestResponse, RawQuoteArgs, }; use http_client::prpc::PrpcClient; -use ra_tls::attestation::VerifiedAttestation; -use ra_tls::attestation::VersionedAttestation; +use ra_tls::attestation::{AttestationVerifier, VerifiedAttestation, VersionedAttestation}; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -24,12 +23,12 @@ pub(crate) fn build_boot_info( use_boottime_mr: bool, vm_config_str: &str, ) -> Result<BootInfo> { - let mode = att.quote.mode(); + let variant = att.quote.variant(); let (tcb_status, advisory_ids) = dstack_verifier::policy_tcb_fields(att); let app_info = att.decode_app_info_ex(use_boottime_mr, vm_config_str)?; ensure_app_id_len(&app_info.app_id)?; Ok(BootInfo { - attestation_mode: mode, + tee_variant: variant, mr_aggregated: app_info.mr_aggregated.to_vec(), os_image_hash: app_info.os_image_hash, mr_system: app_info.mr_system.to_vec(), @@ -50,7 +49,7 @@ pub(crate) fn ensure_app_id_len(app_id: &[u8]) -> Result<()> { Ok(()) } -pub(crate) async fn local_kms_boot_info(pccs_url: Option<&str>) -> Result<BootInfo> { +pub(crate) async fn local_kms_boot_info(verifier: &AttestationVerifier) -> Result<BootInfo> { let response = app_attest(pad64([0u8; 32])) .await .context("Failed to get local KMS attestation")?; @@ -58,7 +57,7 @@ pub(crate) async fn local_kms_boot_info(pccs_url: Option<&str>) -> Result<BootIn .context("Failed to decode local KMS attestation")?; let verified = attestation .into_v1() - .verify(pccs_url) + .verify(verifier) .await .context("Failed to verify local KMS attestation")?; build_boot_info_for_attestation(&verified, false, "") @@ -193,11 +192,14 @@ pub(crate) fn pad64(hash: [u8; 32]) -> Vec<u8> { padded } -pub(crate) async fn ensure_self_kms_allowed(cfg: &KmsConfig) -> Result<()> { +pub(crate) async fn ensure_self_kms_allowed( + cfg: &KmsConfig, + verifier: &AttestationVerifier, +) -> Result<()> { if !cfg.enforce_self_authorization { return Ok(()); } - let boot_info = local_kms_boot_info(cfg.pccs_url.as_deref()) + let boot_info = local_kms_boot_info(verifier) .await .context("failed to build local KMS boot info")?; let response = cfg @@ -214,6 +216,7 @@ pub(crate) async fn ensure_self_kms_allowed(cfg: &KmsConfig) -> Result<()> { pub(crate) async fn ensure_kms_allowed( cfg: &KmsConfig, attestation: &VerifiedAttestation, + verifier: &AttestationVerifier, ) -> Result<()> { let mut boot_info = build_boot_info_for_attestation(attestation, false, "") .context("failed to build KMS boot info from attestation")?; @@ -223,7 +226,7 @@ pub(crate) async fn ensure_kms_allowed( // validates OS image integrity transitively through the RTMR measurement chain. // TODO: remove once all source KMS instances use the unified PHALA_RATLS_ATTESTATION format. if boot_info.os_image_hash.is_empty() { - let local_info = local_kms_boot_info(cfg.pccs_url.as_deref()) + let local_info = local_kms_boot_info(verifier) .await .context("failed to get local KMS boot info for os_image_hash fallback")?; boot_info.os_image_hash = local_info.os_image_hash; @@ -242,6 +245,92 @@ pub(crate) async fn ensure_kms_allowed( #[cfg(test)] mod tests { use super::*; + use crate::config::Webhook; + use ra_tls::attestation::TeeVariant; + use serde_json::Value; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::thread; + + fn boot_info(identity: u8) -> BootInfo { + BootInfo { + tee_variant: TeeVariant::DstackTdx, + mr_aggregated: vec![identity; 32], + os_image_hash: vec![identity.wrapping_add(1); 32], + mr_system: vec![identity.wrapping_add(2); 32], + app_id: vec![identity.wrapping_add(3); 20], + compose_hash: vec![identity.wrapping_add(4); 32], + instance_id: vec![identity.wrapping_add(5); 20], + device_id: vec![identity.wrapping_add(6); 32], + key_provider_info: vec![identity.wrapping_add(7); 32], + tcb_status: "UpToDate".into(), + advisory_ids: vec![], + } + } + + fn serve(responses: Vec<&'static str>) -> (String, thread::JoinHandle<Vec<(String, Value)>>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let handle = thread::spawn(move || { + let mut requests = Vec::new(); + for response in responses { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = Vec::new(); + let mut chunk = [0u8; 4096]; + let header_end = loop { + let read = stream.read(&mut chunk).unwrap(); + assert_ne!(read, 0, "request ended before headers"); + request.extend_from_slice(&chunk[..read]); + if let Some(position) = request.windows(4).position(|part| part == b"\r\n\r\n") + { + break position + 4; + } + }; + let headers = String::from_utf8_lossy(&request[..header_end]).into_owned(); + let content_length = headers + .lines() + .find_map(|line| { + line.split_once(':').and_then(|(name, value)| { + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::<usize>().unwrap()) + }) + }) + .unwrap_or(0); + while request.len() < header_end + content_length { + let read = stream.read(&mut chunk).unwrap(); + assert_ne!(read, 0, "request ended before body"); + request.extend_from_slice(&chunk[..read]); + } + let path = headers + .lines() + .next() + .unwrap() + .split_whitespace() + .nth(1) + .unwrap() + .to_string(); + let body = + serde_json::from_slice(&request[header_end..header_end + content_length]) + .unwrap(); + requests.push((path, body)); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + response.len(), + response + ) + .unwrap(); + } + requests + }); + (format!("http://{address}"), handle) + } + + fn webhook(url: String) -> AuthApi { + AuthApi::Webhook { + webhook: Webhook { url }, + } + } #[test] fn app_id_len_must_be_20_bytes() { @@ -252,4 +341,67 @@ mod tests { Err(err) => assert!(err.to_string().contains("app_id must be 20 bytes")), } } + + #[rocket::async_test] + async fn repeated_authorization_is_never_served_from_a_decision_cache() { + let (url, server) = serve(vec![ + r#"{"isAllowed":true,"gatewayAppId":"gateway","reason":"initial allow"}"#, + r#"{"isAllowed":false,"gatewayAppId":"gateway","reason":"revoked"}"#, + ]); + let auth = webhook(url); + let info = boot_info(1); + + let first = auth.is_app_allowed(&info, false).await.unwrap(); + let repeated = auth.is_app_allowed(&info, false).await.unwrap(); + assert!(first.is_allowed); + assert!(!repeated.is_allowed); + assert_eq!(repeated.reason, "revoked"); + + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].0, "/bootAuth/app"); + assert_eq!(requests[1].0, "/bootAuth/app"); + assert_eq!(requests[0].1, requests[1].1); + } + + #[rocket::async_test] + async fn authorization_scope_preserves_route_and_every_identity_field() { + let (url, server) = serve(vec![ + r#"{"isAllowed":true,"gatewayAppId":"gateway","reason":"app"}"#, + r#"{"isAllowed":true,"gatewayAppId":"gateway","reason":"kms"}"#, + ]); + let auth = webhook(url); + let app = boot_info(10); + let kms = boot_info(20); + + assert!(auth.is_app_allowed(&app, false).await.unwrap().is_allowed); + assert!(auth.is_app_allowed(&kms, true).await.unwrap().is_allowed); + + let requests = server.join().unwrap(); + assert_eq!(requests[0].0, "/bootAuth/app"); + assert_eq!(requests[1].0, "/bootAuth/kms"); + assert_ne!(requests[0].1["appId"], requests[1].1["appId"]); + assert_ne!(requests[0].1["deviceId"], requests[1].1["deviceId"]); + assert_ne!(requests[0].1["osImageHash"], requests[1].1["osImageHash"]); + assert_ne!(requests[0].1["composeHash"], requests[1].1["composeHash"]); + assert_ne!(requests[0].1["instanceId"], requests[1].1["instanceId"]); + assert_ne!(requests[0].1["mrAggregated"], requests[1].1["mrAggregated"]); + } + + #[rocket::async_test] + async fn malformed_backend_response_fails_closed_and_next_request_recovers() { + let (url, server) = serve(vec![ + "not-json", + r#"{"isAllowed":true,"gatewayAppId":"gateway","reason":"recovered"}"#, + ]); + let auth = webhook(url); + let info = boot_info(30); + + let error = auth.is_app_allowed(&info, false).await.unwrap_err(); + assert!(error.to_string().contains("failed to decode response")); + let recovered = auth.is_app_allowed(&info, false).await.unwrap(); + assert!(recovered.is_allowed); + assert_eq!(recovered.reason, "recovered"); + assert_eq!(server.join().unwrap().len(), 2); + } } diff --git a/dstack/kms/src/onboard_service.rs b/dstack/kms/src/onboard_service.rs index 346e458de..2d2d8d0af 100644 --- a/dstack/kms/src/onboard_service.rs +++ b/dstack/kms/src/onboard_service.rs @@ -2,7 +2,10 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::sync::{Arc, Mutex}; +use std::{ + sync::{Arc, Mutex, OnceLock}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; use anyhow::{bail, Context, Result}; use dstack_kms_rpc::{ @@ -19,13 +22,16 @@ use ra_rpc::{ }; use ra_tls::{ attestation::{ - GetDeviceId, PlatformEvidence, QuoteContentType, VerifiedAttestation, VersionedAttestation, + AttestationVerifier, GetDeviceId, PlatformEvidence, QuoteContentType, VerifiedAttestation, + VersionedAttestation, }, cert::{CaCert, CertRequest}, rcgen::{Certificate, KeyPair, PKCS_ECDSA_P256_SHA256}, }; -use safe_write::safe_write; +use safe_write::{safe_write, safe_write_with_mode}; use sha2::Digest; +use tokio::sync::Mutex as AsyncMutex; +use tracing::info; use crate::{ config::KmsConfig, @@ -40,11 +46,31 @@ use crate::{ #[derive(Clone)] pub struct OnboardState { config: KmsConfig, + attestation_verifier: Arc<AttestationVerifier>, + bootstrap_lock: Arc<AsyncMutex<()>>, + shutdown: Arc<OnceLock<rocket::Shutdown>>, } impl OnboardState { - pub fn new(config: KmsConfig) -> Self { - Self { config } + pub fn new(config: KmsConfig) -> Result<Self> { + let attestation_verifier = Arc::new( + AttestationVerifier::load(&config.attestation) + .context("failed to load attestation verifier")?, + ); + Ok(Self { + config, + attestation_verifier, + bootstrap_lock: Arc::new(AsyncMutex::new(())), + shutdown: Arc::new(OnceLock::new()), + }) + } + + /// Hand the Rocket shutdown handle to the service so `finish` can stop the + /// server after its response has been sent. + pub fn set_shutdown(&self, shutdown: rocket::Shutdown) -> Result<()> { + self.shutdown + .set(shutdown) + .map_err(|_| anyhow::anyhow!("onboard shutdown handle is already set")) } } @@ -62,12 +88,37 @@ impl RpcCall<OnboardState> for OnboardHandler { } } +fn validate_onboarding_domain(domain: &str) -> Result<()> { + if domain.is_empty() || domain.len() > 253 || !domain.is_ascii() { + bail!("domain must be a non-empty ASCII DNS name of at most 253 bytes"); + } + for label in domain.split('.') { + if label.is_empty() + || label.len() > 63 + || label.starts_with('-') + || label.ends_with('-') + || !label + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + { + bail!("domain contains an invalid DNS label"); + } + } + Ok(()) +} + impl OnboardRpc for OnboardHandler { async fn bootstrap(self, request: BootstrapRequest) -> Result<BootstrapResponse> { - ensure_self_kms_allowed(&self.state.config) + validate_onboarding_domain(&request.domain)?; + let _bootstrap_guard = self.state.bootstrap_lock.lock().await; + let cfg = &self.state.config; + if cfg.bootstrap_info().exists() || cfg.root_ca_key().exists() || cfg.k256_key().exists() { + bail!("KMS has already been bootstrapped"); + } + ensure_self_kms_allowed(cfg, &self.state.attestation_verifier) .await .context("KMS is not allowed to bootstrap")?; - let keys = Keys::generate(&request.domain) + let keys = Keys::generate(&request.domain, self.state.config.attest_rpc_cert) .await .context("Failed to generate keys")?; @@ -75,7 +126,6 @@ impl OnboardRpc for OnboardHandler { let ca_pubkey = keys.ca_key.public_key_der(); let attestation = attest_keys(&ca_pubkey, &k256_pubkey).await?; - let cfg = &self.state.config; let response = BootstrapResponse { ca_pubkey, k256_pubkey, @@ -88,6 +138,12 @@ impl OnboardRpc for OnboardHandler { } async fn onboard(self, request: OnboardRequest) -> Result<OnboardResponse> { + validate_onboarding_domain(&request.domain)?; + let _bootstrap_guard = self.state.bootstrap_lock.lock().await; + let cfg = &self.state.config; + if cfg.root_ca_key().exists() || cfg.k256_key().exists() { + bail!("KMS has already been onboarded"); + } let source_url = request.source_url.trim_end_matches('/').to_string(); let source_url = if source_url.ends_with("/prpc") { source_url @@ -95,22 +151,19 @@ impl OnboardRpc for OnboardHandler { format!("{source_url}/prpc") }; let keys = Keys::onboard( - &self.state.config, + cfg, &source_url, &request.domain, - self.state.config.pccs_url.clone(), + self.state.attestation_verifier.clone(), ) .await .context("Failed to onboard")?; let k256_pubkey = keys.k256_key.verifying_key().to_sec1_bytes().to_vec(); - keys.store(&self.state.config) - .context("Failed to store keys")?; + keys.store(cfg).context("Failed to store keys")?; Ok(OnboardResponse { k256_pubkey }) } async fn get_attestation_info(self) -> Result<AttestationInfoResponse> { - let pccs_url = self.state.config.pccs_url.clone(); - // Get attestation from guest agent let report_data = pad64([0u8; 32]); let response = app_attest(report_data) @@ -120,7 +173,7 @@ impl OnboardRpc for OnboardHandler { // Decode and verify the attestation to get real device ID let attestation = VersionedAttestation::from_bytes(&response.attestation) .context("Failed to decode attestation")?; - let attestation_mode = match &attestation.clone().into_v1().platform { + let tee_variant = match &attestation.clone().into_v1().platform { PlatformEvidence::Tdx { .. } => "dstack-tdx", PlatformEvidence::SevSnp { .. } => "dstack-amd-sev-snp", PlatformEvidence::GcpTdx { .. } => "dstack-gcp-tdx", @@ -130,7 +183,7 @@ impl OnboardRpc for OnboardHandler { .to_string(); let verified = attestation .into_v1() - .verify(pccs_url.as_deref()) + .verify(&self.state.attestation_verifier) .await .context("Failed to verify attestation")?; @@ -154,7 +207,7 @@ impl OnboardRpc for OnboardHandler { build_attestation_info_response( &verified, - attestation_mode, + tee_variant, &info.vm_config, self.state.config.site_name.clone(), eth_rpc_url, @@ -163,13 +216,21 @@ impl OnboardRpc for OnboardHandler { } async fn finish(self) -> anyhow::Result<()> { - std::process::exit(0); + let shutdown = self + .state + .shutdown + .get() + .context("onboard shutdown handle is unavailable")?; + // Graceful shutdown lets Rocket finish sending this response before the + // server stops, so the client learns that onboarding succeeded. + shutdown.clone().notify(); + Ok(()) } } fn build_attestation_info_response( verified: &VerifiedAttestation, - attestation_mode: String, + tee_variant: String, vm_config: &str, site_name: String, eth_rpc_url: String, @@ -182,7 +243,7 @@ fn build_attestation_info_response( device_id: sha2::Sha256::digest(&raw_device_id).to_vec(), mr_aggregated: boot_info.mr_aggregated, os_image_hash: boot_info.os_image_hash, - attestation_mode, + tee_variant, site_name, eth_rpc_url, kms_contract_address, @@ -243,6 +304,7 @@ mod tests { dstack_types::mr_config::MrConfigV3::new( vec![0x11; 20], vec![0x22; 32], + None, dstack_types::KeyProviderKind::None, Vec::new(), vec![0x99; 20], @@ -328,11 +390,55 @@ mod tests { assert_eq!(response.ppid, vec![0xab; 64]); assert_eq!(response.mr_aggregated.len(), 32); assert_eq!(response.os_image_hash, os_image_hash.to_vec()); - assert_eq!(response.attestation_mode, "dstack-amd-sev-snp"); + assert_eq!(response.tee_variant, "dstack-amd-sev-snp"); assert_eq!(response.site_name, "test-site"); assert_eq!(response.eth_rpc_url, "https://rpc.example"); assert_eq!(response.kms_contract_address, "0x1234"); } + + #[test] + fn onboarding_domain_accepts_dns_name() { + validate_onboarding_domain("kms.example.com").unwrap(); + } + + #[test] + fn onboarding_domain_rejects_empty_overlong_and_invalid_labels() { + let overlong = "a".repeat(254); + for domain in [ + "", + overlong.as_str(), + "-kms.example.com", + "kms-.example.com", + "kms..example.com", + "kms_example.com", + ] { + assert!(validate_onboarding_domain(domain).is_err(), "{domain:?}"); + } + } + + fn ca_cert_expiring_at(not_after: SystemTime) -> Vec<u8> { + let key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap(); + CertRequest::builder() + .subject("Test CA") + .ca_level(0) + .not_after(not_after) + .key(&key) + .build() + .self_signed() + .unwrap() + .pem() + .into_bytes() + } + + #[test] + fn ca_certificate_is_renewed_only_within_the_renewal_window() { + let now = UNIX_EPOCH + Duration::from_secs(2_000_000_000); + let inside_window = ca_cert_expiring_at(now + CA_RENEWAL_WINDOW - Duration::from_secs(1)); + let outside_window = ca_cert_expiring_at(now + CA_RENEWAL_WINDOW + Duration::from_secs(1)); + + assert!(ca_cert_expires_within(&inside_window, now, CA_RENEWAL_WINDOW).unwrap()); + assert!(!ca_cert_expires_within(&outside_window, now, CA_RENEWAL_WINDOW).unwrap()); + } } struct Keys { @@ -347,12 +453,20 @@ struct Keys { } impl Keys { - async fn generate(domain: &str) -> Result<Self> { + async fn generate(domain: &str, attest_rpc_cert: bool) -> Result<Self> { let tmp_ca_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256)?; let ca_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256)?; let rpc_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256)?; let k256_key = SigningKey::random(&mut rand::rngs::OsRng); - Self::from_keys(tmp_ca_key, ca_key, rpc_key, k256_key, domain).await + Self::from_keys( + tmp_ca_key, + ca_key, + rpc_key, + k256_key, + domain, + attest_rpc_cert, + ) + .await } async fn from_keys( @@ -361,6 +475,7 @@ impl Keys { rpc_key: KeyPair, k256_key: SigningKey, domain: &str, + attest_rpc_cert: bool, ) -> Result<Self> { let tmp_ca_cert = CertRequest::builder() .org_name("Dstack") @@ -378,20 +493,32 @@ impl Keys { .key(&ca_key) .build() .self_signed()?; - let pubkey = rpc_key.public_key_der(); - let report_data = QuoteContentType::RaTlsCert.to_report_data(&pubkey); - let response = app_attest(report_data.to_vec()) - .await - .context("Failed to get quote")?; - let attestation = VersionedAttestation::from_bytes(&response.attestation) - .context("Invalid attestation")?; + // The only place the KMS embeds its own attestation. Skipping it lets + // the KMS run outside a TEE for development; it does not affect the + // verification of quotes presented *to* the KMS, which is a separate + // path (main_service::ensure_app_attestation_allowed) and stays on. + let attestation = if attest_rpc_cert { + let pubkey = rpc_key.public_key_der(); + let report_data = QuoteContentType::RaTlsCert.to_report_data(&pubkey); + let response = app_attest(report_data.to_vec()).await.context( + "failed to get a quote for the KMS RPC certificate. The KMS attests \ + itself through the dstack guest agent, so it must run inside a dstack \ + CVM. For local development set attest_rpc_cert = false", + )?; + Some( + VersionedAttestation::from_bytes(&response.attestation) + .context("Invalid attestation")?, + ) + } else { + None + }; // Sign WWW server cert with KMS cert let rpc_cert = CertRequest::builder() .subject(domain) .alt_names(&[domain.to_string()]) .special_usage("kms:rpc") - .maybe_attestation(Some(&attestation)) + .maybe_attestation(attestation.as_ref()) .key(&rpc_key) .build() .signed_by(&ca_cert, &ca_key)?; @@ -411,7 +538,7 @@ impl Keys { cfg: &KmsConfig, other_kms_url: &str, domain: &str, - pccs_url: Option<String>, + attestation_verifier: Arc<AttestationVerifier>, ) -> Result<Self> { let attestation_slot = Arc::new(Mutex::new(None::<VerifiedAttestation>)); let attestation_slot_out = attestation_slot.clone(); @@ -431,22 +558,27 @@ impl Keys { *slot = Some(attestation); Ok(()) })) - .maybe_pccs_url(pccs_url.clone()) + .attestation_verifier(attestation_verifier.clone()) .build() .into_client()?; let mut kms_client = KmsClient::new(client); let tmp_ca = kms_client.get_temp_ca_cert().await?; let (ra_cert, ra_key) = gen_ra_cert(tmp_ca.temp_ca_cert, tmp_ca.temp_ca_key).await?; - let ra_client = RaClient::new_mtls(other_kms_url.into(), ra_cert, ra_key, pccs_url) - .context("Failed to create client")?; + let ra_client = RaClient::new_mtls( + other_kms_url.into(), + ra_cert, + ra_key, + attestation_verifier.clone(), + ) + .context("Failed to create client")?; kms_client = KmsClient::new(ra_client); let source_attestation = attestation_slot .lock() .map_err(|_| anyhow::anyhow!("source attestation mutex poisoned"))? .clone() .context("Missing source KMS attestation")?; - ensure_kms_allowed(cfg, &source_attestation) + ensure_kms_allowed(cfg, &source_attestation, &attestation_verifier) .await .context("Source KMS is not allowed for onboarding")?; @@ -470,7 +602,15 @@ impl Keys { KeyPair::from_pem(&tmp_ca_key_pem).context("Failed to parse tmp CA key")?; let ecdsa_key = SigningKey::from_slice(&root_k256_key).context("Failed to parse ECDSA key")?; - Self::from_keys(tmp_ca_key, ca_key, rpc_key, ecdsa_key, domain).await + Self::from_keys( + tmp_ca_key, + ca_key, + rpc_key, + ecdsa_key, + domain, + cfg.attest_rpc_cert, + ) + .await } fn store(&self, cfg: &KmsConfig) -> Result<()> { @@ -481,10 +621,10 @@ impl Keys { } fn store_keys(&self, cfg: &KmsConfig) -> Result<()> { - safe_write(cfg.tmp_ca_key(), self.tmp_ca_key.serialize_pem())?; - safe_write(cfg.root_ca_key(), self.ca_key.serialize_pem())?; - safe_write(cfg.rpc_key(), self.rpc_key.serialize_pem())?; - safe_write(cfg.k256_key(), self.k256_key.to_bytes())?; + safe_write_with_mode(cfg.tmp_ca_key(), self.tmp_ca_key.serialize_pem(), 0o600)?; + safe_write_with_mode(cfg.root_ca_key(), self.ca_key.serialize_pem(), 0o600)?; + safe_write_with_mode(cfg.rpc_key(), self.rpc_key.serialize_pem(), 0o600)?; + safe_write_with_mode(cfg.k256_key(), self.k256_key.to_bytes(), 0o600)?; Ok(()) } @@ -514,21 +654,73 @@ pub(crate) async fn update_certs(cfg: &KmsConfig) -> Result<()> { let domain = domain.trim(); // Regenerate certificates using existing keys - let keys = Keys::from_keys(tmp_ca_key, ca_key, rpc_key, k256_key, domain) - .await - .context("Failed to regenerate certificates")?; + let keys = Keys::from_keys( + tmp_ca_key, + ca_key, + rpc_key, + k256_key, + domain, + cfg.attest_rpc_cert, + ) + .await + .context("Failed to regenerate certificates")?; + + renew_ca_cert_if_expiring( + cfg.root_ca_cert(), + keys.ca_cert.pem(), + "KMS root CA certificate", + )?; + renew_ca_cert_if_expiring( + cfg.tmp_ca_cert(), + keys.tmp_ca_cert.pem(), + "temporary client CA certificate", + )?; + + // The RPC leaf depends on the refreshed domain and platform attestation, so + // it is reissued on every startup. + safe_write(cfg.rpc_cert(), keys.rpc_cert.pem())?; + info!("Reissued the KMS RPC certificate for {domain}"); - // Write the new certificates to files - keys.store_certs(cfg)?; + Ok(()) +} +const CA_RENEWAL_WINDOW: Duration = Duration::from_secs(365 * 24 * 60 * 60); + +fn renew_ca_cert_if_expiring( + path: impl AsRef<std::path::Path>, + renewed_pem: String, + description: &str, +) -> Result<()> { + let path = path.as_ref(); + let current_pem = fs::read(path) + .with_context(|| format!("Failed to read {description} from {}", path.display()))?; + if !ca_cert_expires_within(¤t_pem, SystemTime::now(), CA_RENEWAL_WINDOW)? { + return Ok(()); + } + safe_write(path, renewed_pem)?; + info!("Renewed {description}"); Ok(()) } -pub(crate) async fn bootstrap_keys(cfg: &KmsConfig) -> Result<()> { - ensure_self_kms_allowed(cfg) +fn ca_cert_expires_within(cert_pem: &[u8], now: SystemTime, window: Duration) -> Result<bool> { + let (_, pem) = + x509_parser::pem::parse_x509_pem(cert_pem).context("Failed to parse CA certificate PEM")?; + let cert = pem.parse_x509().context("Failed to parse CA certificate")?; + let now = now + .duration_since(UNIX_EPOCH) + .context("System time is before the Unix epoch")? + .as_secs(); + let renewal_deadline = now.saturating_add(window.as_secs()); + let not_after = u64::try_from(cert.validity().not_after.timestamp()).unwrap_or(0); + Ok(not_after <= renewal_deadline) +} + +pub(crate) async fn bootstrap_keys(cfg: &KmsConfig, verifier: &AttestationVerifier) -> Result<()> { + validate_onboarding_domain(&cfg.onboard.auto_bootstrap_domain)?; + ensure_self_kms_allowed(cfg, verifier) .await .context("KMS is not allowed to auto-bootstrap")?; - let keys = Keys::generate(&cfg.onboard.auto_bootstrap_domain) + let keys = Keys::generate(&cfg.onboard.auto_bootstrap_domain, cfg.attest_rpc_cert) .await .context("Failed to generate keys")?; keys.store(cfg)?; diff --git a/dstack/kms/src/www/onboard.html b/dstack/kms/src/www/onboard.html index 9f28784ca..a2bc75552 100644 --- a/dstack/kms/src/www/onboard.html +++ b/dstack/kms/src/www/onboard.html @@ -174,8 +174,8 @@ <h1>{{ siteName || 'dstack KMS Setup' }}</h1> <div v-else-if="attestationInfo" class="attestation-info"> <h3>Attestation Info (for on-chain registration)</h3> <div class="info-row"> - <span class="info-label">Attestation Mode:</span> - <span class="info-value">{{ attestationInfo.attestation_mode }}</span> + <span class="info-label">TEE Variant:</span> + <span class="info-value">{{ attestationInfo.tee_variant }}</span> </div> <div class="info-row" v-if="attestationInfo.ppid"> <span class="info-label">PPID:</span> diff --git a/dstack/local-key-provider/build/docker-compose.yaml b/dstack/local-key-provider/build/docker-compose.yaml index 596d6aafe..1e5c37690 100644 --- a/dstack/local-key-provider/build/docker-compose.yaml +++ b/dstack/local-key-provider/build/docker-compose.yaml @@ -22,7 +22,7 @@ services: - "/dev/sgx_enclave:/dev/sgx_enclave" - "/dev/sgx_provision:/dev/sgx_provision" volumes: - - "./sgx_default_qcnl.conf:/etc/sgx_default_qcnl.conf" + - "/etc/sgx_default_qcnl.conf:/etc/sgx_default_qcnl.conf:ro" - "aesmd:/var/run/aesmd/" network_mode: "host" diff --git a/dstack/local-key-provider/build/sgx_default_qcnl.conf b/dstack/local-key-provider/build/sgx_default_qcnl.conf deleted file mode 100644 index 0f184cdc0..000000000 --- a/dstack/local-key-provider/build/sgx_default_qcnl.conf +++ /dev/null @@ -1,9 +0,0 @@ -{ - "pccs_url": "https://pccs.phala.network/sgx/certification/v4/", - "use_secure_cert": true, - "retry_times": 6, - "retry_delay": 10, - "pck_cache_expire_hours": 168, - "verify_collateral_cache_expire_hours": 168, - "local_cache_only": false -} diff --git a/dstack/lspci/Cargo.toml b/dstack/lspci/Cargo.toml index bbb6f35cd..00b33724c 100644 --- a/dstack/lspci/Cargo.toml +++ b/dstack/lspci/Cargo.toml @@ -14,3 +14,4 @@ anyhow.workspace = true [dev-dependencies] insta.workspace = true +tempfile.workspace = true diff --git a/dstack/lspci/src/lib.rs b/dstack/lspci/src/lib.rs index ef3565ede..10c84e70e 100644 --- a/dstack/lspci/src/lib.rs +++ b/dstack/lspci/src/lib.rs @@ -6,6 +6,8 @@ use std::process::Command; use anyhow::{Context, Result}; +pub mod sysfs; + /// Represents a PCI device with the specified fields. #[derive(Debug)] pub struct Device { diff --git a/dstack/lspci/src/sysfs.rs b/dstack/lspci/src/sysfs.rs new file mode 100644 index 000000000..b866a28fc --- /dev/null +++ b/dstack/lspci/src/sysfs.rs @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! GPU inventory read straight from sysfs. +//! +//! Distinct from the `lspci` parser in this crate: no subprocess, no text +//! parsing, and it sees devices the NVIDIA driver never bound. That matters in +//! two places with opposite failure policies -- the boot attestation gate must +//! fail closed when the inventory cannot be read, while a telemetry collector +//! wants to shrug and report no GPUs -- so this returns the raw counts and lets +//! each caller decide. + +use std::path::Path; + +use anyhow::{Context, Result}; + +/// Where the kernel exposes the PCI bus. +pub const PCI_DEVICES: &str = "/sys/bus/pci/devices"; + +const NVIDIA_VENDOR_ID: &str = "0x10de"; +/// PCI class prefixes for VGA and 3D controllers. +const DISPLAY_CLASS_PREFIXES: [&str; 2] = ["0x0300", "0x0302"]; + +/// Display-class PCI devices, split by vendor. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct GpuInventory { + /// All display-class devices, whatever the vendor. + pub total: u32, + /// The subset made by NVIDIA. + pub nvidia: u32, +} + +impl GpuInventory { + /// True when at least one NVIDIA display device is attached. + pub fn has_nvidia(&self) -> bool { + self.nvidia > 0 + } +} + +/// Counts display-class GPUs on the live PCI bus. +pub fn gpu_inventory() -> Result<GpuInventory> { + gpu_inventory_at(Path::new(PCI_DEVICES)) +} + +/// Counts display-class GPUs under an arbitrary sysfs root, for tests. +pub fn gpu_inventory_at(devices_path: &Path) -> Result<GpuInventory> { + let entries = std::fs::read_dir(devices_path) + .with_context(|| format!("failed to enumerate {}", devices_path.display()))?; + let mut inventory = GpuInventory::default(); + for entry in entries { + let device = entry.context("failed to read PCI device entry")?; + let class_path = device.path().join("class"); + let class = std::fs::read_to_string(&class_path) + .with_context(|| format!("failed to read {}", class_path.display()))?; + if !class + .trim() + .get(..6) + .is_some_and(|prefix| DISPLAY_CLASS_PREFIXES.contains(&prefix)) + { + continue; + } + inventory.total += 1; + let vendor_path = device.path().join("vendor"); + let vendor = std::fs::read_to_string(&vendor_path) + .with_context(|| format!("failed to read {}", vendor_path.display()))?; + if vendor.trim() == NVIDIA_VENDOR_ID { + inventory.nvidia += 1; + } + } + Ok(inventory) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn add_pci_device(root: &Path, name: &str, vendor: &str, class: &str) { + let dir = root.join(name); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("vendor"), vendor).unwrap(); + std::fs::write(dir.join("class"), class).unwrap(); + } + + #[test] + fn inventory_counts_nvidia_and_non_nvidia_gpus() { + let root = tempfile::tempdir().unwrap(); + add_pci_device(root.path(), "0000:01:00.0", "0x10de\n", "0x030200\n"); + add_pci_device(root.path(), "0000:02:00.0", "0x1234\n", "0x030000\n"); + // A virtio NIC is not a display device and must not be counted. + add_pci_device(root.path(), "0000:03:00.0", "0x1af4\n", "0x020000\n"); + assert_eq!( + gpu_inventory_at(root.path()).unwrap(), + GpuInventory { + total: 2, + nvidia: 1 + } + ); + } + + /// The common CVM: virtio devices only. `has_nvidia` is the gate that keeps + /// GPU telemetry from costing such a guest anything. + #[test] + fn a_guest_without_a_display_device_reports_no_gpus() { + let root = tempfile::tempdir().unwrap(); + add_pci_device(root.path(), "0000:01:00.0", "0x1af4\n", "0x020000\n"); + add_pci_device(root.path(), "0000:02:00.0", "0x1af4\n", "0x010000\n"); + let inventory = gpu_inventory_at(root.path()).unwrap(); + assert_eq!(inventory, GpuInventory::default()); + assert!(!inventory.has_nvidia()); + } + + /// Callers must be able to tell "no GPUs" from "could not look". + #[test] + fn an_unreadable_root_is_an_error_not_an_empty_inventory() { + assert!(gpu_inventory_at(Path::new("/nonexistent/pci/devices")).is_err()); + } +} diff --git a/dstack/nsm-qvl/src/lib.rs b/dstack/nsm-qvl/src/lib.rs index c8144dde1..e1afd8df2 100644 --- a/dstack/nsm-qvl/src/lib.rs +++ b/dstack/nsm-qvl/src/lib.rs @@ -48,6 +48,48 @@ pub struct NsmCollateral { /// Fingerprint: 64:1A:03:21:A3:E2:44:EF:E4:56:46:31:95:D6:06:31:7E:D7:CD:CC:3C:17:56:E0:98:93:F3:C6:8F:79:BB:5B pub const AWS_NITRO_ENCLAVES_ROOT_G1: &str = include_str!("../certs/AWS_NitroEnclaves_Root-G1.pem"); +/// Verifies NSM attestation documents against a caller-selected trust anchor. +/// +/// This mirrors `dcap_qvl::verify::QuoteVerifier`: use [`Self::new_prod`] for +/// AWS production or [`Self::new`] to inject a PEM encoded root certificate. +#[derive(Debug, Clone)] +pub struct QuoteVerifier { + root_ca_pem: String, +} + +impl QuoteVerifier { + pub fn new(root_ca_pem: impl Into<String>) -> Self { + Self { + root_ca_pem: root_ca_pem.into(), + } + } + + pub fn new_prod() -> Self { + Self::new(AWS_NITRO_ENCLAVES_ROOT_G1) + } + + pub fn root_ca_pem(&self) -> &str { + &self.root_ca_pem + } + + pub fn verify( + &self, + cose_sign1_bytes: &[u8], + collateral: Option<&NsmCollateral>, + now: Option<std::time::SystemTime>, + ) -> Result<NsmVerifiedReport> { + verify_attestation(cose_sign1_bytes, &self.root_ca_pem, collateral, now) + } + + pub async fn fetch_and_verify( + &self, + cose_sign1_bytes: &[u8], + now: Option<std::time::SystemTime>, + ) -> Result<NsmVerifiedReport> { + collateral::get_collateral_and_verify(cose_sign1_bytes, &self.root_ca_pem, now).await + } +} + /// Parsed COSE Sign1 structure for NSM attestation #[derive(Debug)] pub struct CoseSign1 { diff --git a/dstack/nvattest/Cargo.toml b/dstack/nvattest/Cargo.toml new file mode 100644 index 000000000..672209a92 --- /dev/null +++ b/dstack/nvattest/Cargo.toml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "nvattest" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "Shared wrapper around the NVIDIA nvattest CLI used by the boot gate and the guest agent" + +[dependencies] +anyhow.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = ["std"] } +tokio = { workspace = true, features = ["process", "time"] } +tracing.workspace = true +url.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/dstack/nvattest/src/lib.rs b/dstack/nvattest/src/lib.rs new file mode 100644 index 000000000..f933ecd66 --- /dev/null +++ b/dstack/nvattest/src/lib.rs @@ -0,0 +1,423 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Thin wrapper around the NVIDIA `nvattest` CLI. +//! +//! Both the boot path (`dstack-util setup`) and the guest agent's on-demand +//! `AttestGpu` RPC shell out to the same binary with the same argument shape, +//! so the invocation, the proxy-URL rules and the output checks live here +//! rather than being written twice with two sets of bugs. +//! +//! What this crate does *not* do is decide what the evidence means. Appraisal +//! against a policy is the caller's, because the boot gate and a runtime +//! liveness check want different answers from the same bytes. + +use std::{path::Path, process::Output, time::Duration}; + +use anyhow::{bail, Context, Result}; +use serde::Deserialize; +use serde_json::Value; +use tracing::info; + +/// The nvattest binary, installed into the rootfs by the nvattest recipe. +pub const NVATTEST: &str = "/usr/bin/nvattest"; + +/// NVIDIA's relying-party policy that tolerates a cached OCSP responder nonce. +/// Only used when a collateral proxy is configured. +pub const TRUST_OUTPOST_POLICY: &str = "/usr/share/nvattest/policies/allow_trust_outpost_ocsp.rego"; + +/// SPDM fixes the GPU evidence nonce at 32 bytes. The SDK rejects anything +/// else, so callers get a clear error here instead of a CLI parse failure. +pub const NONCE_LEN: usize = 32; + +/// Long enough for a cold collateral fetch on a slow link, short enough that a +/// wedged driver cannot hang the caller forever. +pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(300); + +/// Raw `nvattest attest --format json` output plus the nonce it answered. +#[derive(Debug)] +pub struct Attestation { + /// Exact stdout bytes. Callers that bind evidence to a measurement must + /// hash these, not a re-serialization of the parsed claims. + pub output: Vec<u8>, + /// Hex-encoded nonce passed to nvattest, echoed back as `eat_nonce`. + pub nonce: String, + /// Parsed claims, one per attested device. + pub claims: Vec<Value>, +} + +#[derive(Deserialize)] +struct NvattestOutput { + result_code: i64, + claims: Vec<Value>, +} + +#[derive(Deserialize)] +struct CollectEvidenceOutput { + result_code: i64, + #[serde(default)] + evidences: Vec<Value>, +} + +#[derive(Deserialize)] +struct NonceClaim { + #[serde(rename = "eat_nonce")] + eat_nonce: String, + #[serde(rename = "x-nvidia-gpu-attestation-report-nonce-match")] + nonce_match: bool, +} + +/// True when this image can attest a GPU at all. +pub fn available() -> bool { + Path::new(NVATTEST).exists() +} + +/// Validate a collateral proxy URL. Rejects anything carrying credentials or a +/// path, so a misconfigured value cannot smuggle a different endpoint past the +/// `{proxy}/ocsp` and `{proxy}/v1/rim/...` construction below. +pub fn normalize_proxy_url(proxy_url: Option<&str>) -> Result<Option<String>> { + let Some(proxy_url) = proxy_url.map(str::trim).filter(|url| !url.is_empty()) else { + return Ok(None); + }; + let parsed = url::Url::parse(proxy_url).context("invalid NVIDIA attestation proxy URL")?; + if !matches!(parsed.scheme(), "http" | "https") || parsed.host_str().is_none() { + bail!("NVIDIA attestation proxy must be an absolute HTTP(S) URL"); + } + if parsed.query().is_some() + || parsed.fragment().is_some() + || !parsed.username().is_empty() + || parsed.password().is_some() + || parsed.path() != "/" + { + bail!( + "NVIDIA attestation proxy URL must not contain credentials, path, query, or fragment" + ); + } + Ok(Some(parsed.as_str().trim_end_matches('/').to_string())) +} + +/// Build the CLI arguments for a local (self-verifying) GPU attestation. +pub fn args(nonce: &str, proxy_url: Option<&str>) -> Result<Vec<String>> { + let mut args = vec![ + "attest".to_string(), + "--device".to_string(), + "gpu".to_string(), + "--verifier".to_string(), + "local".to_string(), + "--nonce".to_string(), + nonce.to_string(), + "--format".to_string(), + "json".to_string(), + ]; + if let Some(proxy_url) = normalize_proxy_url(proxy_url)? { + args.extend([ + "--ocsp-url".to_string(), + format!("{proxy_url}/ocsp"), + "--rim-url".to_string(), + proxy_url, + "--relying-party-policy".to_string(), + TRUST_OUTPOST_POLICY.to_string(), + ]); + } + Ok(args) +} + +/// Run a command with a bounded timeout, killing the child if it expires so a +/// wedged GPU tool cannot outlive the caller that gave up on it. +pub async fn run_command(program: &str, args: &[&str], timeout: Duration) -> Result<Output> { + let child = tokio::process::Command::new(program) + .args(args) + .kill_on_drop(true) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .with_context(|| format!("failed to run {program}"))?; + tokio::time::timeout(timeout, child.wait_with_output()) + .await + .with_context(|| format!("{program} timed out"))? + .with_context(|| format!("failed to run {program}")) +} + +/// Run `nvattest` against `nonce` and return its raw output. +/// +/// `nonce` must be exactly [`NONCE_LEN`] bytes; it is passed through verbatim +/// so a caller can compare its own challenge against `eat_nonce` without +/// reversing any transform. +/// +/// Verifies only that nvattest succeeded and that every claim answers this +/// nonce. Everything else -- device counts, CC state, application policy -- +/// is appraisal, and belongs to the caller. +pub async fn attest( + nonce: &[u8], + proxy_url: Option<&str>, + timeout: Duration, +) -> Result<Attestation> { + let (nonce, output) = run(nonce, proxy_url, timeout).await?; + check_status(&output)?; + let claims = check_nonce(&output.stdout, &nonce)?; + Ok(Attestation { + output: output.stdout, + nonce, + claims, + }) +} + +/// Run `nvattest` and hand back its raw output *whatever its exit status*, +/// along with the hex nonce it was given. +/// +/// Errors only when the tool could not be run at all. The boot gate persists +/// stdout before judging the status, so a failed appraisal still leaves +/// evidence on disk to debug; that is why the status check is separate. +pub async fn run( + nonce: &[u8], + proxy_url: Option<&str>, + timeout: Duration, +) -> Result<(String, Output)> { + if !available() { + bail!("nvattest is not available in this image"); + } + let nonce = check_nonce_len(nonce)?; + let args = args(&nonce, proxy_url)?; + if args.iter().any(|arg| arg == "--relying-party-policy") + && !Path::new(TRUST_OUTPOST_POLICY).is_file() + { + bail!("NVIDIA attestation proxy is configured but {TRUST_OUTPOST_POLICY} is missing"); + } + let borrowed = args.iter().map(String::as_str).collect::<Vec<_>>(); + let output = run_command(NVATTEST, &borrowed, timeout).await?; + if !output.stderr.is_empty() { + info!("nvattest: {}", truncated_lossy(&output.stderr, 2048)); + } + Ok((nonce, output)) +} + +/// Collect GPU-signed evidence for `nonce` without appraising it. +/// +/// Returns the `evidences` array alone, which is the shape `attest +/// --gpu-evidence-source=file` expects, and the shape a third party needs: the +/// report and certificate chain, not a verdict about them. +pub async fn collect_evidence(nonce: &[u8], timeout: Duration) -> Result<(String, Vec<u8>)> { + let nonce = check_nonce_len(nonce)?; + let args = [ + "collect-evidence", + "--device", + "gpu", + "--nonce", + &nonce, + "--format", + "json", + ]; + let output = run_command(NVATTEST, &args, timeout).await?; + if !output.stderr.is_empty() { + info!( + "nvattest collect-evidence: {}", + truncated_lossy(&output.stderr, 2048) + ); + } + check_status(&output)?; + let parsed: CollectEvidenceOutput = serde_json::from_slice(&output.stdout) + .context("failed to parse nvattest collect-evidence output")?; + if parsed.result_code != 0 { + bail!( + "nvattest collect-evidence failed (result_code={})", + parsed.result_code + ); + } + if parsed.evidences.is_empty() { + bail!("nvattest collected no GPU evidence"); + } + let evidences = + serde_json::to_vec(&parsed.evidences).context("failed to re-encode GPU evidence")?; + Ok((nonce, evidences)) +} + +fn check_nonce_len(nonce: &[u8]) -> Result<String> { + if nonce.len() != NONCE_LEN { + bail!( + "gpu attestation nonce must be {NONCE_LEN} bytes, got {}", + nonce.len() + ); + } + Ok(hex_encode(nonce)) +} + +/// Turn a non-zero nvattest exit into an error carrying a bounded stderr tail. +pub fn check_status(output: &Output) -> Result<()> { + if !output.status.success() { + bail!( + "nvattest exited with {}: {}", + output.status, + truncated_lossy(&output.stderr, 512), + ); + } + Ok(()) +} + +/// Require a successful result and one fresh claim per device answering `nonce`. +pub fn check_nonce(stdout: &[u8], nonce: &str) -> Result<Vec<Value>> { + let parsed: NvattestOutput = + serde_json::from_slice(stdout).context("failed to parse nvattest JSON output")?; + if parsed.result_code != 0 { + bail!( + "nvattest JSON result is not successful (result_code={})", + parsed.result_code + ); + } + if parsed.claims.is_empty() { + bail!("nvattest returned no GPU claims"); + } + for (index, claim) in parsed.claims.iter().enumerate() { + let nonce_claim: NonceClaim = serde_json::from_value(claim.clone()) + .with_context(|| format!("invalid GPU claim at index {index}"))?; + if nonce_claim.eat_nonce != nonce || !nonce_claim.nonce_match { + bail!("gpu claim at index {index} does not answer the requested nonce"); + } + } + Ok(parsed.claims) +} + +fn hex_encode(bytes: &[u8]) -> String { + use std::fmt::Write; + bytes + .iter() + .fold(String::with_capacity(bytes.len() * 2), |mut out, b| { + let _ = write!(out, "{b:02x}"); + out + }) +} + +/// Truncate on a character boundary so a huge or binary stderr cannot flood logs. +pub fn truncated_lossy(bytes: &[u8], limit: usize) -> String { + let text = String::from_utf8_lossy(bytes); + match text.char_indices().nth(limit) { + Some((end, _)) => format!("{}...", &text[..end]), + None => text.into_owned(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn claim(nonce: &str, matched: bool) -> String { + format!( + r#"{{"eat_nonce":"{nonce}","x-nvidia-gpu-attestation-report-nonce-match":{matched}}}"# + ) + } + + fn output(result_code: i64, claims: &[String]) -> Vec<u8> { + format!( + r#"{{"result_code":{result_code},"claims":[{}]}}"#, + claims.join(",") + ) + .into_bytes() + } + + #[test] + fn accepts_claims_answering_the_requested_nonce() { + let nonce = "aa".repeat(NONCE_LEN); + let out = output(0, &[claim(&nonce, true), claim(&nonce, true)]); + assert_eq!(check_nonce(&out, &nonce).unwrap().len(), 2); + } + + #[test] + fn rejects_a_claim_answering_a_different_nonce() { + let nonce = "aa".repeat(NONCE_LEN); + let stale = "bb".repeat(NONCE_LEN); + let out = output(0, &[claim(&nonce, true), claim(&stale, true)]); + let err = check_nonce(&out, &nonce).unwrap_err().to_string(); + assert!(err.contains("does not answer the requested nonce"), "{err}"); + } + + #[test] + fn rejects_a_claim_whose_device_reported_no_nonce_match() { + let nonce = "aa".repeat(NONCE_LEN); + let out = output(0, &[claim(&nonce, false)]); + assert!(check_nonce(&out, &nonce).is_err()); + } + + #[test] + fn rejects_unsuccessful_and_empty_results() { + let nonce = "aa".repeat(NONCE_LEN); + assert!(check_nonce(&output(1, &[claim(&nonce, true)]), &nonce).is_err()); + assert!(check_nonce(&output(0, &[]), &nonce).is_err()); + } + + #[tokio::test] + async fn collect_evidence_rejects_a_nonce_of_the_wrong_length() { + let err = collect_evidence(&[0u8; 16], DEFAULT_TIMEOUT) + .await + .unwrap_err() + .to_string(); + assert!( + err.contains("32 bytes") || err.contains("not available"), + "{err}" + ); + } + + #[tokio::test] + async fn rejects_a_nonce_of_the_wrong_length() { + let err = attest(&[0u8; 16], None, DEFAULT_TIMEOUT) + .await + .unwrap_err() + .to_string(); + // Length is checked before the binary is, so this holds off-target too. + assert!( + err.contains("32 bytes") || err.contains("not available"), + "{err}" + ); + } + + #[test] + fn proxy_url_rules() { + assert_eq!(normalize_proxy_url(None).unwrap(), None); + assert_eq!(normalize_proxy_url(Some(" ")).unwrap(), None); + assert_eq!( + normalize_proxy_url(Some("http://10.0.2.2:8090/")).unwrap(), + Some("http://10.0.2.2:8090".to_string()) + ); + for bad in [ + "ftp://host/", + "file:///tmp/proxy", + "https://user@example.com", + "http://user:pw@host/", + "https://example.com/base", + "https://example.com?q=1", + "not-a-url", + ] { + assert!( + normalize_proxy_url(Some(bad)).is_err(), + "{bad} must be rejected" + ); + } + } + + #[test] + fn proxy_routes_ocsp_and_rim_and_selects_outpost_policy() { + let proxied = args("aa", Some("http://10.0.2.2:8090/")).unwrap(); + assert!(proxied + .windows(2) + .any(|w| w == ["--ocsp-url", "http://10.0.2.2:8090/ocsp"])); + assert!(proxied + .windows(2) + .any(|w| w == ["--rim-url", "http://10.0.2.2:8090"])); + assert!(proxied + .windows(2) + .any(|w| w == ["--relying-party-policy", TRUST_OUTPOST_POLICY])); + + let direct = args("aa", None).unwrap(); + assert!(!direct.iter().any(|a| a == "--ocsp-url")); + assert!(!direct.iter().any(|a| a == "--relying-party-policy")); + } + + #[test] + fn truncation_is_bounded_and_utf8_safe() { + assert_eq!(truncated_lossy(b"abc", 10), "abc"); + assert_eq!(truncated_lossy(b"abcdef", 3), "abc..."); + assert_eq!( + truncated_lossy("\u{4f60}\u{597d}".as_bytes(), 1), + "\u{4f60}..." + ); + } +} diff --git a/dstack/nvidia-attest-proxy/Cargo.toml b/dstack/nvidia-attest-proxy/Cargo.toml new file mode 100644 index 000000000..fbde446a7 --- /dev/null +++ b/dstack/nvidia-attest-proxy/Cargo.toml @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "dstack-nvidia-attest-proxy" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +anyhow.workspace = true +base64.workspace = true +bytes.workspace = true +futures.workspace = true +chrono.workspace = true +clap = { workspace = true, features = ["env"] } +dashmap.workspace = true +hex = { workspace = true, features = ["alloc"] } +http.workspace = true +http-body-util.workspace = true +humantime.workspace = true +hyper = { workspace = true, features = ["server"] } +hyper-util = { workspace = true, features = ["tokio"] } +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +tokio = { workspace = true, features = ["full"] } +tracing.workspace = true +tracing-subscriber.workspace = true +url.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/dstack/nvidia-attest-proxy/README.md b/dstack/nvidia-attest-proxy/README.md new file mode 100644 index 000000000..ae0caf6d7 --- /dev/null +++ b/dstack/nvidia-attest-proxy/README.md @@ -0,0 +1,78 @@ +# NVIDIA GPU Attestation Proxy + +`dstack-nvidia-attest-proxy` is a persistent, PCCS-like cache for the two NVIDIA +services used by local GPU attestation: + +- `POST /ocsp` caches DER OCSP responses by the request's certificate IDs. The + per-request OCSP nonce is deliberately excluded from the key. +- `GET /v1/rim/<id>` caches version-addressed NVIDIA RIM documents. +- `GET /healthz` reports process health. +- `GET /info` reports fresh/stale entry counts per cache kind. + +The proxy never signs or rewrites collateral. `nvattest` in the CVM still +verifies NVIDIA's certificate chains, OCSP signatures, response validity +window, RIM signatures, measurements, and the fresh GPU evidence nonce. Cache +entries are persisted so a warm cache survives both proxy and VM restarts. + +## Run + +```bash +cargo build --release -p dstack-nvidia-attest-proxy +sudo install -m 0755 target/release/dstack-nvidia-attest-proxy /usr/local/bin/ +sudo install -d -m 0750 /var/cache/dstack/nvidia-attest-proxy + +sudo /usr/local/bin/dstack-nvidia-attest-proxy \ + --listen 0.0.0.0:8090 \ + --cache-dir /var/cache/dstack/nvidia-attest-proxy +``` + +All options have environment-variable equivalents; see `--help`. In +particular, `NV_ATTESTATION_SERVICE_KEY` supplies an optional NVIDIA bearer +token without placing it on the command line. + +Configure the VMM with a URL reachable during early guest boot: + +```toml +[cvm] +nvidia_attestation_proxy_url = "http://10.0.2.2:8090" +``` + +For QEMU user-mode networking, `10.0.2.2` is normally the host address. Other +networking modes should use the corresponding host or fleet service address. + +## Cache behavior + +On a cold miss the original request is forwarded to NVIDIA. A successful OCSP +response is cached no later than its signed `nextUpdate` and no longer than +`--ocsp-max-ttl` (24 hours by default). Responses without `nextUpdate` use one +hour from `thisUpdate`, matching the pinned NVIDIA SDK. RIM documents default +to a 30-day TTL. + +Two mechanisms keep entries warm, and they compose: a background sweep (every +`--refresh-interval`, 10 minutes by default) renews entries that have +consumed half their lifetime, so a warm cache rides through an NVIDIA outage +with close to a full validity window instead of only the remainder; entries +past their usefulness are dropped, not retried. As a synchronous fallback, an +OCSP response with less than `--ocsp-refresh-before` validity remaining +(five minutes by default) is refreshed in-line on the next request before it +is served — concurrent refreshes for the same entry are coalesced. If an +in-line refresh fails, the proxy keeps serving the old response only until +its existing signed expiry; it never extends or serves an expired one. + +Expired OCSP entries are never served. Therefore a warm cache removes the +NVIDIA service from the boot path only for the signed validity period; it does +not turn revocation checking into an indefinite fail-open. RIM documents are +signed and version-addressed, so an expired RIM entry — unlike OCSP — may +still be served for up to `--rim-max-stale` (7 days by default) when the +upstream is unreachable or failing; the guest verifies its signature either +way. Response headers expose `X-Dstack-Cache: HIT|MISS|REFRESH|STALE`, `Age`, +and `X-Dstack-Cache-Expires` for operations. +Each cache kind is capped at 10,000 entries by default; the oldest entry is +evicted when the limit is reached. Use `--max-cache-entries-per-kind` to tune +the bound for a deployment. + +Because a cached response contains the nonce from the request that populated +the cache, `nvattest` reports `x-nvidia-cert-ocsp-nonce-matches = false` on a +hit. When the proxy URL is configured, dstack selects NVIDIA's packaged +`allow_trust_outpost_ocsp.rego` policy. This relaxes only the OCSP nonce check; +the GPU attestation report's independent nonce remains required by dstack. diff --git a/dstack/nvidia-attest-proxy/src/cache.rs b/dstack/nvidia-attest-proxy/src/cache.rs new file mode 100644 index 000000000..7f03212e9 --- /dev/null +++ b/dstack/nvidia-attest-proxy/src/cache.rs @@ -0,0 +1,419 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use bytes::Bytes; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tokio::fs; +use tokio::sync::Mutex; + +#[derive(Debug, Clone)] +pub(crate) struct CacheEntry { + pub(crate) body: Bytes, + pub(crate) content_type: String, + pub(crate) fetched_at: i64, + pub(crate) expires_at: i64, +} + +impl CacheEntry { + pub(crate) fn is_fresh(&self, now: i64) -> bool { + self.expires_at > now + } +} + +/// What populated a cache entry. Lets the background refresher re-run the +/// exact upstream query and lets operators identify entries on disk (the +/// filenames themselves are hashes). +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum EntrySource { + RimId(String), + OcspRequestB64(String), +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct CacheMetadata { + pub(crate) content_type: String, + pub(crate) fetched_at: i64, + pub(crate) expires_at: i64, + pub(crate) body_sha256: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) source: Option<EntrySource>, +} + +#[derive(Debug, Clone)] +pub(crate) struct CacheStore { + root: PathBuf, + max_entries_per_namespace: usize, + maintenance: std::sync::Arc<Mutex<()>>, +} + +impl CacheStore { + pub(crate) async fn new( + root: impl Into<PathBuf>, + max_entries_per_namespace: usize, + ) -> Result<Self> { + let root = root.into(); + fs::create_dir_all(root.join("ocsp")) + .await + .with_context(|| format!("failed to create {}", root.join("ocsp").display()))?; + fs::create_dir_all(root.join("rim")) + .await + .with_context(|| format!("failed to create {}", root.join("rim").display()))?; + Ok(Self { + root, + max_entries_per_namespace, + maintenance: Default::default(), + }) + } + + /// Read an entry. Entries expired for longer than `max_stale_secs` are + /// removed and reported as absent; entries expired within that window are + /// returned so callers can decide whether stale data is acceptable + /// (check with [`CacheEntry::is_fresh`]). + pub(crate) async fn get( + &self, + namespace: &str, + key: &str, + now: i64, + max_stale_secs: i64, + ) -> Result<Option<CacheEntry>> { + let (body_path, metadata_path) = self.paths(namespace, key); + let metadata = match fs::read(&metadata_path).await { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(error) + .with_context(|| format!("failed to read {}", metadata_path.display())) + } + }; + let metadata: CacheMetadata = match serde_json::from_slice(&metadata) { + Ok(metadata) => metadata, + Err(error) => { + tracing::warn!(path = %metadata_path.display(), %error, "discarding invalid cache metadata"); + self.remove_if(namespace, key, |current| { + !matches!(current, ReadState::Valid(_)) + }) + .await; + return Ok(None); + } + }; + if metadata.expires_at.saturating_add(max_stale_secs) <= now { + self.remove_if(namespace, key, move |current| match current { + ReadState::Valid(metadata) => { + metadata.expires_at.saturating_add(max_stale_secs) <= now + } + _ => true, + }) + .await; + return Ok(None); + } + let body = match fs::read(&body_path).await { + Ok(body) => body, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + self.remove_if(namespace, key, |current| { + !matches!(current, ReadState::Valid(_)) + }) + .await; + return Ok(None); + } + Err(error) => { + return Err(error) + .with_context(|| format!("failed to read {}", body_path.display())) + } + }; + if hex::encode(Sha256::digest(&body)) != metadata.body_sha256 { + tracing::warn!(path = %body_path.display(), "discarding cache entry with an invalid digest"); + self.remove_if(namespace, key, |current| { + !matches!(current, ReadState::Valid(_)) + }) + .await; + return Ok(None); + } + Ok(Some(CacheEntry { + body: Bytes::from(body), + content_type: metadata.content_type, + fetched_at: metadata.fetched_at, + expires_at: metadata.expires_at, + })) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) async fn put( + &self, + namespace: &str, + key: &str, + body: &[u8], + content_type: &str, + fetched_at: i64, + expires_at: i64, + source: Option<EntrySource>, + ) -> Result<()> { + let (body_path, metadata_path) = self.paths(namespace, key); + let metadata = CacheMetadata { + content_type: content_type.to_string(), + fetched_at, + expires_at, + body_sha256: hex::encode(Sha256::digest(body)), + source, + }; + let metadata = + serde_json::to_vec(&metadata).context("failed to serialize cache metadata")?; + + let _maintenance = self.maintenance.lock().await; + self.evict_for_insert(namespace, key).await?; + // Metadata is the commit marker: readers ignore an entry until both + // the body and its matching metadata have been atomically renamed. + atomic_write(&body_path, body).await?; + atomic_write(&metadata_path, &metadata).await?; + Ok(()) + } + + /// List the metadata of every entry in a namespace, for the background + /// refresher and the `/info` endpoint. Unreadable metadata is skipped. + pub(crate) async fn list(&self, namespace: &str) -> Result<Vec<(String, CacheMetadata)>> { + let directory = self.root.join(namespace); + let mut entries = fs::read_dir(&directory) + .await + .with_context(|| format!("failed to scan {}", directory.display()))?; + let mut listed = Vec::new(); + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + if path.extension().and_then(|ext| ext.to_str()) != Some("json") { + continue; + } + let Some(key) = path.file_stem().and_then(|stem| stem.to_str()) else { + continue; + }; + let Some(metadata) = fs::read(&path) + .await + .ok() + .and_then(|data| serde_json::from_slice::<CacheMetadata>(&data).ok()) + else { + continue; + }; + listed.push((key.to_string(), metadata)); + } + Ok(listed) + } + + /// Remove an entry, but only when a re-read under the maintenance lock + /// still satisfies `should_remove`. This prevents a stale read from + /// deleting an entry that a concurrent `put` just refreshed. + async fn remove_if<F>(&self, namespace: &str, key: &str, should_remove: F) + where + F: FnOnce(ReadState) -> bool, + { + let _maintenance = self.maintenance.lock().await; + let (body_path, metadata_path) = self.paths(namespace, key); + let state = match fs::read(&metadata_path).await { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return, + Err(_) => ReadState::Corrupt, + Ok(data) => match serde_json::from_slice::<CacheMetadata>(&data) { + Err(_) => ReadState::Corrupt, + Ok(metadata) => match fs::read(&body_path).await { + Err(_) => ReadState::Corrupt, + Ok(body) if hex::encode(Sha256::digest(&body)) != metadata.body_sha256 => { + ReadState::Corrupt + } + Ok(_) => ReadState::Valid(metadata), + }, + }, + }; + if should_remove(state) { + let _ = fs::remove_file(body_path).await; + let _ = fs::remove_file(metadata_path).await; + } + } + + async fn evict_for_insert(&self, namespace: &str, new_key: &str) -> Result<()> { + let directory = self.root.join(namespace); + let mut entries = fs::read_dir(&directory) + .await + .with_context(|| format!("failed to scan {}", directory.display()))?; + let mut metadata = Vec::new(); + let mut replacing = false; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + if path.extension().and_then(|ext| ext.to_str()) != Some("json") { + continue; + } + let Some(key) = path.file_stem().and_then(|stem| stem.to_str()) else { + continue; + }; + replacing |= key == new_key; + let fetched_at = fs::read(&path) + .await + .ok() + .and_then(|data| serde_json::from_slice::<CacheMetadata>(&data).ok()) + .map(|metadata| metadata.fetched_at) + .unwrap_or(i64::MIN); + metadata.push((fetched_at, key.to_string())); + } + if replacing { + return Ok(()); + } + metadata.sort_unstable(); + let remove_count = metadata + .len() + .saturating_add(1) + .saturating_sub(self.max_entries_per_namespace); + for (_, key) in metadata.into_iter().take(remove_count) { + let (body_path, metadata_path) = self.paths(namespace, &key); + let _ = fs::remove_file(body_path).await; + let _ = fs::remove_file(metadata_path).await; + } + Ok(()) + } + + fn paths(&self, namespace: &str, key: &str) -> (PathBuf, PathBuf) { + // Callers pass SHA-256 hex keys, so filenames never contain input from + // a URL or certificate ID directly. + let base = self.root.join(namespace).join(key); + (base.with_extension("body"), base.with_extension("json")) + } +} + +enum ReadState { + Valid(CacheMetadata), + Corrupt, +} + +async fn atomic_write(path: &Path, value: &[u8]) -> Result<()> { + let nonce = rand_suffix(); + let temporary = path.with_extension(format!("tmp-{nonce}")); + fs::write(&temporary, value) + .await + .with_context(|| format!("failed to write {}", temporary.display()))?; + fs::rename(&temporary, path).await.with_context(|| { + format!( + "failed to rename {} to {}", + temporary.display(), + path.display() + ) + })?; + Ok(()) +} + +fn rand_suffix() -> String { + use std::sync::atomic::{AtomicU64, Ordering}; + static SEQUENCE: AtomicU64 = AtomicU64::new(0); + format!( + "{}-{}", + std::process::id(), + SEQUENCE.fetch_add(1, Ordering::Relaxed) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn persists_and_expires_entries() { + let dir = tempfile::tempdir().unwrap(); + let cache = CacheStore::new(dir.path(), 10).await.unwrap(); + cache + .put( + "ocsp", + "abcd", + b"response", + "application/ocsp-response", + 100, + 200, + None, + ) + .await + .unwrap(); + + let reopened = CacheStore::new(dir.path(), 10).await.unwrap(); + let entry = reopened.get("ocsp", "abcd", 150, 0).await.unwrap().unwrap(); + assert_eq!(entry.body, &b"response"[..]); + assert_eq!(entry.fetched_at, 100); + assert_eq!(entry.expires_at, 200); + assert!(reopened + .get("ocsp", "abcd", 200, 0) + .await + .unwrap() + .is_none()); + } + + #[tokio::test] + async fn stale_entries_survive_within_the_stale_window() { + let dir = tempfile::tempdir().unwrap(); + let cache = CacheStore::new(dir.path(), 10).await.unwrap(); + cache + .put("rim", "abcd", b"doc", "application/json", 100, 200, None) + .await + .unwrap(); + + // Within the stale window the entry is returned but not fresh. + let entry = cache.get("rim", "abcd", 250, 100).await.unwrap().unwrap(); + assert!(!entry.is_fresh(250)); + // Beyond the stale window it is removed. + assert!(cache.get("rim", "abcd", 301, 100).await.unwrap().is_none()); + assert!(cache.get("rim", "abcd", 150, 100).await.unwrap().is_none()); + } + + #[tokio::test] + async fn rejects_modified_cache_bodies() { + let dir = tempfile::tempdir().unwrap(); + let cache = CacheStore::new(dir.path(), 10).await.unwrap(); + cache + .put("rim", "abcd", b"one", "application/json", 100, 200, None) + .await + .unwrap(); + fs::write(dir.path().join("rim/abcd.body"), b"two") + .await + .unwrap(); + assert!(cache.get("rim", "abcd", 150, 0).await.unwrap().is_none()); + } + + #[tokio::test] + async fn evicts_oldest_entry_at_capacity() { + let dir = tempfile::tempdir().unwrap(); + let cache = CacheStore::new(dir.path(), 1).await.unwrap(); + cache + .put("rim", "old", b"old", "application/json", 100, 1000, None) + .await + .unwrap(); + cache + .put("rim", "new", b"new", "application/json", 200, 1000, None) + .await + .unwrap(); + assert!(cache.get("rim", "old", 300, 0).await.unwrap().is_none()); + assert_eq!( + cache.get("rim", "new", 300, 0).await.unwrap().unwrap().body, + &b"new"[..] + ); + } + + #[tokio::test] + async fn lists_entries_with_sources() { + let dir = tempfile::tempdir().unwrap(); + let cache = CacheStore::new(dir.path(), 10).await.unwrap(); + cache + .put( + "rim", + "abcd", + b"doc", + "application/json", + 100, + 200, + Some(EntrySource::RimId("NV_GPU_DRIVER_GH100_580.1".into())), + ) + .await + .unwrap(); + let listed = cache.list("rim").await.unwrap(); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].0, "abcd"); + assert_eq!( + listed[0].1.source, + Some(EntrySource::RimId("NV_GPU_DRIVER_GH100_580.1".into())) + ); + } +} diff --git a/dstack/nvidia-attest-proxy/src/lib.rs b/dstack/nvidia-attest-proxy/src/lib.rs new file mode 100644 index 000000000..f91ff9bfe --- /dev/null +++ b/dstack/nvidia-attest-proxy/src/lib.rs @@ -0,0 +1,1214 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +mod cache; +mod ocsp; + +use std::{ + convert::Infallible, + net::SocketAddr, + path::PathBuf, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{bail, Context, Result}; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; +use bytes::Bytes; +use cache::{CacheEntry, CacheStore, EntrySource}; +use dashmap::DashMap; +use futures::StreamExt; +use http::{ + header::{ACCEPT, CACHE_CONTROL, CONTENT_TYPE}, + HeaderValue, Method, Request, Response, StatusCode, +}; +use http_body_util::{BodyExt, Full, Limited}; +use hyper::{body::Incoming, server::conn::http1, service::service_fn}; +use hyper_util::rt::{TokioIo, TokioTimer}; +use reqwest::Client; +use sha2::{Digest, Sha256}; +use tokio::{net::TcpListener, sync::Mutex, time::MissedTickBehavior}; +use tracing::{debug, info, warn}; +use url::Url; + +const MAX_OCSP_REQUEST_BYTES: usize = 64 * 1024; +const MAX_OCSP_RESPONSE_BYTES: usize = 1024 * 1024; +const MAX_RIM_RESPONSE_BYTES: usize = 16 * 1024 * 1024; +const OCSP_CONTENT_TYPE: &str = "application/ocsp-response"; +const JSON_CONTENT_TYPE: &str = "application/json"; +const HEADER_READ_TIMEOUT: Duration = Duration::from_secs(10); +/// Concurrent upstream requests per background refresh sweep. +const REFRESH_CONCURRENCY: usize = 4; + +#[derive(Debug, Clone)] +pub struct ProxyConfig { + pub listen_addr: SocketAddr, + pub cache_dir: PathBuf, + pub ocsp_url: Url, + pub rim_url: Url, + pub service_key: Option<String>, + pub request_timeout: Duration, + pub connect_timeout: Duration, + pub ocsp_max_ttl: Duration, + pub ocsp_default_ttl: Duration, + pub ocsp_refresh_before: Duration, + pub rim_ttl: Duration, + /// How long an expired RIM document may still be served when the upstream + /// cannot provide a fresh copy. RIM documents are signed and + /// version-addressed, so staleness is an availability trade-off, not a + /// security one. Zero disables stale serving. + pub rim_max_stale: Duration, + /// Interval between background refresh sweeps. Entries past half their + /// lifetime are renewed so a warm cache rides through upstream outages + /// with close to a full validity window. Zero disables the refresher. + pub refresh_interval: Duration, + pub max_cache_entries_per_kind: usize, +} + +impl ProxyConfig { + pub fn validate(&self) -> Result<()> { + validate_upstream_url("OCSP", &self.ocsp_url)?; + validate_upstream_url("RIM", &self.rim_url)?; + for (name, value) in [ + ("request timeout", self.request_timeout), + ("connect timeout", self.connect_timeout), + ("OCSP maximum TTL", self.ocsp_max_ttl), + ("OCSP default TTL", self.ocsp_default_ttl), + ("OCSP refresh window", self.ocsp_refresh_before), + ("RIM TTL", self.rim_ttl), + ] { + if value.is_zero() { + bail!("{name} must be greater than zero"); + } + if value.as_secs() > i64::MAX as u64 { + bail!("{name} is too large"); + } + } + // These two may be zero, meaning the feature is disabled. + for (name, value) in [ + ("RIM maximum staleness", self.rim_max_stale), + ("refresh interval", self.refresh_interval), + ] { + if value.as_secs() > i64::MAX as u64 { + bail!("{name} is too large"); + } + } + if self.max_cache_entries_per_kind == 0 { + bail!("maximum cache entries per kind must be greater than zero"); + } + Ok(()) + } +} + +#[derive(Debug, Clone)] +struct Proxy { + client: Client, + cache: CacheStore, + config: Arc<ProxyConfig>, + fills: Arc<DashMap<String, Arc<Mutex<()>>>>, +} + +#[derive(Debug)] +struct UpstreamResponse { + status: StatusCode, + content_type: String, + body: Bytes, +} + +pub async fn run(config: ProxyConfig) -> Result<()> { + config.validate()?; + let listen_addr = config.listen_addr; + let proxy = Proxy::new(config).await?; + let listener = TcpListener::bind(listen_addr) + .await + .with_context(|| format!("failed to bind {listen_addr}"))?; + info!(%listen_addr, "GPU attestation collateral proxy listening"); + tokio::spawn(proxy.clone().refresh_loop()); + + loop { + let (stream, peer_addr) = listener + .accept() + .await + .context("failed to accept connection")?; + let proxy = proxy.clone(); + tokio::spawn(async move { + let io = TokioIo::new(stream); + if let Err(error) = http1::Builder::new() + .timer(TokioTimer::new()) + .header_read_timeout(HEADER_READ_TIMEOUT) + .serve_connection( + io, + service_fn(move |request| { + let proxy = proxy.clone(); + async move { Ok::<_, Infallible>(proxy.handle(request).await) } + }), + ) + .await + { + tracing::debug!(%peer_addr, %error, "HTTP connection failed"); + } + }); + } +} + +impl Proxy { + async fn new(config: ProxyConfig) -> Result<Self> { + config.validate()?; + let cache = CacheStore::new(&config.cache_dir, config.max_cache_entries_per_kind).await?; + let client = Client::builder() + .connect_timeout(config.connect_timeout) + .timeout(config.request_timeout) + .redirect(reqwest::redirect::Policy::none()) + .user_agent(concat!( + "dstack-nvidia-attest-proxy/", + env!("CARGO_PKG_VERSION") + )) + .build() + .context("failed to create upstream HTTP client")?; + Ok(Self { + client, + cache, + config: Arc::new(config), + fills: Arc::new(DashMap::new()), + }) + } + + fn rim_max_stale_secs(&self) -> i64 { + self.config.rim_max_stale.as_secs().min(i64::MAX as u64) as i64 + } + + async fn handle(&self, request: Request<Incoming>) -> Response<Full<Bytes>> { + match (request.method(), request.uri().path()) { + (&Method::GET, "/healthz") => text_response(StatusCode::OK, "ok\n"), + (&Method::GET, "/info") => self.handle_info().await, + (&Method::POST, "/ocsp") => { + if !request + .headers() + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value.split(';').next().is_some_and(|value| { + value + .trim() + .eq_ignore_ascii_case("application/ocsp-request") + }) + }) + { + return text_response( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + "expected application/ocsp-request\n", + ); + } + let body = match Limited::new(request.into_body(), MAX_OCSP_REQUEST_BYTES) + .collect() + .await + { + Ok(body) => body.to_bytes(), + Err(error) => { + tracing::debug!(%error, "failed to read OCSP request"); + return text_response( + StatusCode::PAYLOAD_TOO_LARGE, + "invalid or oversized OCSP request\n", + ); + } + }; + self.handle_ocsp(body).await + } + (&Method::GET, path) if path.starts_with("/v1/rim/") => { + let rim_id = &path["/v1/rim/".len()..]; + if !valid_rim_id(rim_id) || request.uri().query().is_some() { + return text_response(StatusCode::BAD_REQUEST, "invalid RIM ID\n"); + } + self.handle_rim(rim_id).await + } + (&Method::GET, "/ocsp") | (&Method::POST, "/healthz") | (&Method::POST, "/info") => { + text_response(StatusCode::METHOD_NOT_ALLOWED, "method not allowed\n") + } + _ => text_response(StatusCode::NOT_FOUND, "not found\n"), + } + } + + async fn handle_ocsp(&self, body: Bytes) -> Response<Full<Bytes>> { + let key = match ocsp::request_cache_key(&body) { + Ok(key) => key, + Err(error) => { + tracing::debug!(%error, "rejected malformed OCSP request"); + return text_response(StatusCode::BAD_REQUEST, "invalid OCSP request\n"); + } + }; + // `cache_get(.., 0)` only ever returns entries that have not yet hit + // their hard expiry (max_stale_secs=0 for OCSP). An entry can still + // be fresh but close enough to expiry to need a synchronous renewal + // (`ocsp_needs_refresh`) before it is served again. + if let Some(entry) = self.cache_get("ocsp", &key, 0).await { + if !self.ocsp_needs_refresh(&entry) { + return cached_response(entry, "HIT"); + } + } + + let (fill_key, fill) = self.fill_lock("ocsp", &key); + let response = { + let _guard = fill.lock().await; + self.fill_ocsp(&key, body).await + }; + drop(fill); + self.release_fill(&fill_key); + response + } + + /// Resolve an OCSP request, synchronously renewing a cached entry once it + /// enters its refresh window. A failed renewal falls back to the still- + /// valid cached response instead of failing the request — the goal is to + /// avoid the *next* request finding a hard-expired cache, not to punish + /// this one for a transient upstream hiccup. + async fn fill_ocsp(&self, key: &str, body: Bytes) -> Response<Full<Bytes>> { + let cached = self.cache_get("ocsp", key, 0).await; + if let Some(entry) = &cached { + if !self.ocsp_needs_refresh(entry) { + // A concurrent request already renewed this entry while we + // were waiting for the fill lock. + return cached_response(entry.clone(), "HIT"); + } + } + let upstream = match self.fetch_ocsp(body.clone()).await { + Ok(response) => response, + Err(error) => { + warn!(%error, "OCSP upstream request failed"); + if let Some(entry) = cached { + return cached_response(entry, "HIT"); + } + return text_response(StatusCode::BAD_GATEWAY, "OCSP upstream unavailable\n"); + } + }; + if upstream.status != StatusCode::OK { + if let Some(entry) = &cached { + warn!(status = %upstream.status, "OCSP refresh returned an error; using valid cached response"); + return cached_response(entry.clone(), "HIT"); + } + } + let cache_state = if cached.is_some() { "REFRESH" } else { "MISS" }; + if upstream.status == StatusCode::OK { + let now = now_epoch(); + match ocsp::response_cache_expiry( + &upstream.body, + now, + self.config.ocsp_max_ttl.as_secs(), + self.config.ocsp_default_ttl.as_secs(), + ) { + Ok(expires_at) => { + let source = EntrySource::OcspRequestB64(BASE64.encode(&body)); + if let Err(error) = self + .cache + .put( + "ocsp", + key, + &upstream.body, + &upstream.content_type, + now, + expires_at, + Some(source), + ) + .await + { + warn!(%error, "failed to persist OCSP cache entry"); + } + } + Err(error) => { + warn!(%error, "OCSP response is not cacheable; returning it without caching"); + } + } + } + upstream_response(upstream, cache_state) + } + + async fn handle_rim(&self, rim_id: &str) -> Response<Full<Bytes>> { + let key = hex::encode(Sha256::digest(rim_id.as_bytes())); + let max_stale = self.rim_max_stale_secs(); + if let Some(entry) = self.cache_get("rim", &key, max_stale).await { + if entry.is_fresh(now_epoch()) { + return cached_response(entry, "HIT"); + } + } + + let (fill_key, fill) = self.fill_lock("rim", &key); + let response = { + let _guard = fill.lock().await; + self.fill_rim(&key, rim_id).await + }; + drop(fill); + self.release_fill(&fill_key); + response + } + + /// Fetch a RIM document, falling back to a stale cache entry (bounded by + /// `rim_max_stale`) when the upstream is unreachable or failing. RIM + /// documents are signed and version-addressed, so a stale copy is + /// verifiable and identical to a fresh one; expired OCSP entries are + /// deliberately never served this way. + async fn fill_rim(&self, key: &str, rim_id: &str) -> Response<Full<Bytes>> { + let cached = self.cache_get("rim", key, self.rim_max_stale_secs()).await; + if let Some(entry) = &cached { + if entry.is_fresh(now_epoch()) { + return cached_response(entry.clone(), "HIT"); + } + } + let upstream = match self.fetch_rim(rim_id).await { + Ok(upstream) => { + if upstream.status.is_server_error() { + if let Some(entry) = cached { + warn!(%rim_id, status = %upstream.status, "RIM upstream failing; serving stale document"); + return cached_response(entry, "STALE"); + } + } + upstream + } + Err(error) => { + warn!(%error, %rim_id, "RIM upstream request failed"); + if let Some(entry) = cached { + warn!(%rim_id, "serving stale RIM document while the upstream is unreachable"); + return cached_response(entry, "STALE"); + } + return text_response(StatusCode::BAD_GATEWAY, "RIM upstream unavailable\n"); + } + }; + if upstream.status == StatusCode::OK { + let now = now_epoch(); + let expires_at = now.saturating_add(self.config.rim_ttl.as_secs() as i64); + if let Err(error) = self + .cache + .put( + "rim", + key, + &upstream.body, + &upstream.content_type, + now, + expires_at, + Some(EntrySource::RimId(rim_id.to_string())), + ) + .await + { + warn!(%error, "failed to persist RIM cache entry"); + } + } + upstream_response(upstream, "MISS") + } + + async fn handle_info(&self) -> Response<Full<Bytes>> { + let now = now_epoch(); + let mut report = String::new(); + for namespace in ["ocsp", "rim"] { + match self.cache.list(namespace).await { + Ok(entries) => { + let fresh = entries + .iter() + .filter(|(_, metadata)| metadata.expires_at > now) + .count(); + let stale = entries.len() - fresh; + report.push_str(&format!("{namespace}: {fresh} fresh, {stale} stale\n")); + } + Err(error) => { + report.push_str(&format!("{namespace}: unavailable ({error:#})\n")); + } + } + } + binary_response( + StatusCode::OK, + "text/plain; charset=utf-8", + Bytes::from(report), + ) + } + + /// Renew entries past half their lifetime so a warm cache approaches a + /// full validity window of outage tolerance. The half-life trigger keeps + /// short-lived entries from being polled on every sweep, and entries past + /// their usefulness (OCSP: expiry; RIM: expiry plus the stale window) are + /// dropped instead of being retried forever. + async fn refresh_loop(self) { + let interval = self.config.refresh_interval; + if interval.is_zero() { + info!("background refresh is disabled"); + return; + } + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(MissedTickBehavior::Delay); + loop { + ticker.tick().await; + self.refresh_once().await; + } + } + + async fn refresh_once(&self) { + for namespace in ["ocsp", "rim"] { + let entries = match self.cache.list(namespace).await { + Ok(entries) => entries, + Err(error) => { + warn!(%error, %namespace, "failed to list cache entries for refresh"); + continue; + } + }; + let now = now_epoch(); + let due = entries + .into_iter() + .filter(|(_, metadata)| { + should_refresh(metadata.fetched_at, metadata.expires_at, now) + }) + .collect::<Vec<_>>(); + futures::stream::iter(due) + .for_each_concurrent(REFRESH_CONCURRENCY, |(key, metadata)| async move { + self.refresh_entry(namespace, key, metadata.source).await; + }) + .await; + } + } + + async fn refresh_entry( + &self, + namespace: &'static str, + key: String, + source: Option<EntrySource>, + ) { + let (fill_key, fill) = self.fill_lock(namespace, &key); + { + let _guard = fill.lock().await; + self.refresh_locked(namespace, &key, source).await; + } + drop(fill); + self.release_fill(&fill_key); + } + + async fn refresh_locked(&self, namespace: &str, key: &str, source: Option<EntrySource>) { + let max_stale = match namespace { + "rim" => self.rim_max_stale_secs(), + _ => 0, + }; + // This get drops entries past their usefulness as a side effect, so + // dead entries are cleaned up instead of being retried forever. + let Some(entry) = self.cache_get(namespace, key, max_stale).await else { + return; + }; + if !should_refresh(entry.fetched_at, entry.expires_at, now_epoch()) { + // A concurrent miss already renewed this entry. + return; + } + // Entries from a pre-source cache format cannot be refreshed; they + // serve until they expire naturally. + match &source { + None => {} + Some(EntrySource::RimId(rim_id)) => match self.fetch_rim(rim_id).await { + Ok(upstream) if upstream.status == StatusCode::OK => { + let now = now_epoch(); + let expires_at = now.saturating_add(self.config.rim_ttl.as_secs() as i64); + match self + .cache + .put( + namespace, + key, + &upstream.body, + &upstream.content_type, + now, + expires_at, + source.clone(), + ) + .await + { + Ok(()) => debug!(%rim_id, "refreshed RIM document"), + Err(error) => warn!(%error, "failed to persist refreshed RIM entry"), + } + } + Ok(upstream) => { + debug!(%rim_id, status = %upstream.status, "RIM refresh got an error status") + } + Err(error) => debug!(%rim_id, %error, "RIM refresh failed"), + }, + Some(EntrySource::OcspRequestB64(encoded)) => { + let Ok(request) = BASE64.decode(encoded) else { + debug!(%key, "cache entry has an undecodable OCSP refresh request"); + return; + }; + match self.fetch_ocsp(Bytes::from(request)).await { + Ok(upstream) if upstream.status == StatusCode::OK => { + let now = now_epoch(); + match ocsp::response_cache_expiry( + &upstream.body, + now, + self.config.ocsp_max_ttl.as_secs(), + self.config.ocsp_default_ttl.as_secs(), + ) { + Ok(expires_at) => { + match self + .cache + .put( + namespace, + key, + &upstream.body, + &upstream.content_type, + now, + expires_at, + source.clone(), + ) + .await + { + Ok(()) => debug!(%key, "refreshed OCSP response"), + Err(error) => { + warn!(%error, "failed to persist refreshed OCSP entry") + } + } + } + Err(error) => { + debug!(%error, "refreshed OCSP response is not cacheable") + } + } + } + Ok(upstream) => { + debug!(%key, status = %upstream.status, "OCSP refresh got an error status") + } + Err(error) => debug!(%key, %error, "OCSP refresh failed"), + } + } + } + } + + async fn fetch_ocsp(&self, body: Bytes) -> Result<UpstreamResponse> { + let mut request = self + .client + .post(self.config.ocsp_url.clone()) + .header(CONTENT_TYPE, "application/ocsp-request") + .header(ACCEPT, OCSP_CONTENT_TYPE) + .body(body); + if let Some(service_key) = self.config.service_key.as_deref() { + request = request.bearer_auth(service_key); + } + collect_upstream( + request + .send() + .await + .context("failed to query OCSP upstream")?, + MAX_OCSP_RESPONSE_BYTES, + ) + .await + } + + async fn fetch_rim(&self, rim_id: &str) -> Result<UpstreamResponse> { + let mut url = self.config.rim_url.clone(); + { + let mut segments = url + .path_segments_mut() + .map_err(|_| anyhow::anyhow!("RIM upstream URL cannot be a base URL"))?; + segments.pop_if_empty().extend(["v1", "rim", rim_id]); + } + let mut request = self.client.get(url).header(ACCEPT, JSON_CONTENT_TYPE); + if let Some(service_key) = self.config.service_key.as_deref() { + request = request.bearer_auth(service_key); + } + collect_upstream( + request + .send() + .await + .context("failed to query RIM upstream")?, + MAX_RIM_RESPONSE_BYTES, + ) + .await + } + + async fn cache_get( + &self, + namespace: &str, + key: &str, + max_stale_secs: i64, + ) -> Option<CacheEntry> { + match self + .cache + .get(namespace, key, now_epoch(), max_stale_secs) + .await + { + Ok(entry) => entry, + Err(error) => { + warn!(%error, %namespace, "failed to read cache entry"); + None + } + } + } + + /// Whether an OCSP entry has entered its synchronous refresh window + /// (`ocsp_refresh_before` of its signed expiry). This is a softer signal + /// than hard freshness: an entry that fails this check is still valid + /// and safe to serve as a fallback if renewal fails. + fn ocsp_needs_refresh(&self, entry: &CacheEntry) -> bool { + entry.expires_at.saturating_sub(now_epoch()) + <= self.config.ocsp_refresh_before.as_secs() as i64 + } + + fn fill_lock(&self, namespace: &str, key: &str) -> (String, Arc<Mutex<()>>) { + let fill_key = format!("{namespace}:{key}"); + let lock = self + .fills + .entry(fill_key.clone()) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone(); + (fill_key, lock) + } + + /// Drop a fill lock nobody is holding, so `fills` does not grow with every + /// distinct key ever requested. Waiters hold their own `Arc` clone, which + /// keeps the strong count above one until they are done. + fn release_fill(&self, fill_key: &str) { + self.fills + .remove_if(fill_key, |_, lock| Arc::strong_count(lock) == 1); + } +} + +/// Refresh once an entry has consumed half its lifetime. This both renews +/// long-lived entries well before expiry and avoids polling upstream on every +/// sweep for entries with short validity windows. +fn should_refresh(fetched_at: i64, expires_at: i64, now: i64) -> bool { + let lifetime = expires_at.saturating_sub(fetched_at); + lifetime > 0 && now.saturating_sub(fetched_at) >= lifetime / 2 +} + +async fn collect_upstream(response: reqwest::Response, limit: usize) -> Result<UpstreamResponse> { + if response + .content_length() + .is_some_and(|length| length > limit as u64) + { + bail!("upstream response exceeds {limit} bytes"); + } + let status = response.status(); + let content_type = response + .headers() + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or("application/octet-stream") + .to_string(); + let body = response + .bytes() + .await + .context("failed to read upstream response")?; + if body.len() > limit { + bail!("upstream response exceeds {limit} bytes"); + } + Ok(UpstreamResponse { + status, + content_type, + body, + }) +} + +fn validate_upstream_url(name: &str, url: &Url) -> Result<()> { + if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() { + bail!("{name} upstream URL must be an absolute HTTP(S) URL"); + } + if url.query().is_some() + || url.fragment().is_some() + || !url.username().is_empty() + || url.password().is_some() + { + bail!("{name} upstream URL must not contain credentials, query, or fragment"); + } + Ok(()) +} + +fn valid_rim_id(rim_id: &str) -> bool { + !rim_id.is_empty() + && rim_id.len() <= 256 + && rim_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.')) +} + +fn cached_response(entry: CacheEntry, state: &'static str) -> Response<Full<Bytes>> { + let age = now_epoch().saturating_sub(entry.fetched_at); + let mut response = binary_response(StatusCode::OK, &entry.content_type, entry.body); + response + .headers_mut() + .insert("x-dstack-cache", HeaderValue::from_static(state)); + if let Ok(age) = HeaderValue::from_str(&age.to_string()) { + response.headers_mut().insert("age", age); + } + if let Ok(expires_at) = HeaderValue::from_str(&entry.expires_at.to_string()) { + response + .headers_mut() + .insert("x-dstack-cache-expires", expires_at); + } + response +} + +fn upstream_response( + upstream: UpstreamResponse, + cache_state: &'static str, +) -> Response<Full<Bytes>> { + let mut response = binary_response(upstream.status, &upstream.content_type, upstream.body); + response + .headers_mut() + .insert("x-dstack-cache", HeaderValue::from_static(cache_state)); + response +} + +fn binary_response(status: StatusCode, content_type: &str, body: Bytes) -> Response<Full<Bytes>> { + let mut response = Response::new(Full::new(body)); + *response.status_mut() = status; + response + .headers_mut() + .insert(CACHE_CONTROL, HeaderValue::from_static("no-store")); + if let Ok(content_type) = HeaderValue::from_str(content_type) { + response.headers_mut().insert(CONTENT_TYPE, content_type); + } + response +} + +fn text_response(status: StatusCode, body: &'static str) -> Response<Full<Bytes>> { + binary_response( + status, + "text/plain; charset=utf-8", + Bytes::from_static(body.as_bytes()), + ) +} + +fn now_epoch() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64 +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use chrono::{TimeZone, Utc}; + + use super::*; + + fn der(tag: u8, value: &[u8]) -> Vec<u8> { + let mut result = vec![tag]; + if value.len() < 128 { + result.push(value.len() as u8); + } else { + let bytes = value.len().to_be_bytes(); + let first = bytes.iter().position(|byte| *byte != 0).unwrap(); + result.push(0x80 | (bytes.len() - first) as u8); + result.extend_from_slice(&bytes[first..]); + } + result.extend_from_slice(value); + result + } + + fn sequence(parts: &[Vec<u8>]) -> Vec<u8> { + der(0x30, &parts.concat()) + } + + fn ocsp_request(nonce: u8) -> Bytes { + let algorithm = sequence(&[der(0x06, &[0x2b, 0x0e, 0x03, 0x02, 0x1a])]); + let cert_id = sequence(&[ + algorithm, + der(0x04, &[1; 20]), + der(0x04, &[2; 20]), + der(0x02, &[3]), + ]); + let request_list = sequence(&[sequence(&[cert_id])]); + let extensions = der(0xa2, &sequence(&[der(0x04, &[nonce; 16])])); + Bytes::from(sequence(&[sequence(&[request_list, extensions])])) + } + + fn ocsp_response(now: i64) -> Bytes { + ocsp_response_with_ttl(now, 3600) + } + + fn ocsp_response_with_ttl(now: i64, ttl: i64) -> Bytes { + let format = |timestamp| { + Utc.timestamp_opt(timestamp, 0) + .unwrap() + .format("%Y%m%d%H%M%SZ") + .to_string() + }; + let cert_id = sequence(&[der(0x02, &[1])]); + let single = sequence(&[ + cert_id, + der(0x80, &[]), + der(0x18, format(now - 60).as_bytes()), + der(0xa0, &der(0x18, format(now + ttl).as_bytes())), + ]); + let response_data = sequence(&[ + der(0x82, &[7; 20]), + der(0x18, format(now).as_bytes()), + sequence(&[single]), + ]); + let basic = sequence(&[ + response_data, + sequence(&[der(0x06, &[0x2a, 0x03])]), + der(0x03, &[0]), + ]); + let response_bytes = sequence(&[ + der( + 0x06, + &[0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x01, 0x01], + ), + der(0x04, &basic), + ]); + Bytes::from(sequence(&[der(0x0a, &[0]), der(0xa0, &response_bytes)])) + } + + async fn mock_upstream( + response: Bytes, + content_type: &'static str, + ) -> (Url, Arc<AtomicUsize>, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let requests = Arc::new(AtomicUsize::new(0)); + let request_counter = requests.clone(); + let task = tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + let response = response.clone(); + let request_counter = request_counter.clone(); + tokio::spawn(async move { + http1::Builder::new() + .serve_connection( + TokioIo::new(stream), + service_fn(move |_request| { + let response = response.clone(); + let request_counter = request_counter.clone(); + async move { + request_counter.fetch_add(1, Ordering::SeqCst); + Ok::<_, Infallible>(binary_response( + StatusCode::OK, + content_type, + response, + )) + } + }), + ) + .await + .unwrap(); + }); + } + }); + ( + Url::parse(&format!("http://{address}/ocsp")).unwrap(), + requests, + task, + ) + } + + /// A URL that refuses connections: bind an ephemeral port, then drop it. + async fn dead_upstream() -> Url { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + drop(listener); + Url::parse(&format!("http://{address}/")).unwrap() + } + + fn test_config(cache_dir: PathBuf, ocsp_url: Url, rim_url: Url) -> ProxyConfig { + ProxyConfig { + listen_addr: "127.0.0.1:0".parse().unwrap(), + cache_dir, + ocsp_url, + rim_url, + service_key: None, + request_timeout: Duration::from_secs(5), + connect_timeout: Duration::from_secs(1), + ocsp_max_ttl: Duration::from_secs(86_400), + ocsp_default_ttl: Duration::from_secs(3600), + ocsp_refresh_before: Duration::from_secs(300), + rim_ttl: Duration::from_secs(86_400), + rim_max_stale: Duration::from_secs(7 * 86_400), + refresh_interval: Duration::from_secs(600), + max_cache_entries_per_kind: 10, + } + } + + #[tokio::test] + async fn ocsp_cache_reuses_response_across_nonces_and_restarts() { + let now = now_epoch(); + let response = ocsp_response(now); + let (upstream_url, requests, upstream) = + mock_upstream(response.clone(), OCSP_CONTENT_TYPE).await; + let cache_dir = tempfile::tempdir().unwrap(); + let config = test_config( + cache_dir.path().to_path_buf(), + upstream_url.clone(), + Url::parse("https://rim.attestation.nvidia.com").unwrap(), + ); + + let proxy = Proxy::new(config.clone()).await.unwrap(); + let nonce = std::process::id() as u8; + let miss = proxy.handle_ocsp(ocsp_request(nonce)).await; + assert_eq!(miss.headers()["x-dstack-cache"], "MISS"); + assert_eq!( + miss.into_body().collect().await.unwrap().to_bytes(), + response + ); + let hit = proxy.handle_ocsp(ocsp_request(nonce.wrapping_add(1))).await; + assert_eq!(hit.headers()["x-dstack-cache"], "HIT"); + assert_eq!(requests.load(Ordering::SeqCst), 1); + + drop(proxy); + let reopened = Proxy::new(config).await.unwrap(); + let hit = reopened + .handle_ocsp(ocsp_request(nonce.wrapping_add(2))) + .await; + assert_eq!(hit.headers()["x-dstack-cache"], "HIT"); + assert_eq!(requests.load(Ordering::SeqCst), 1); + upstream.abort(); + } + + #[tokio::test] + async fn ocsp_cache_refreshes_before_expiry() { + let now = now_epoch(); + let response = ocsp_response_with_ttl(now, 60); + let (upstream_url, requests, upstream) = mock_upstream(response, OCSP_CONTENT_TYPE).await; + let cache_dir = tempfile::tempdir().unwrap(); + let proxy = Proxy::new(ProxyConfig { + listen_addr: "127.0.0.1:0".parse().unwrap(), + cache_dir: cache_dir.path().to_path_buf(), + ocsp_url: upstream_url, + rim_url: Url::parse("https://rim.attestation.nvidia.com").unwrap(), + service_key: None, + request_timeout: Duration::from_secs(5), + connect_timeout: Duration::from_secs(1), + ocsp_max_ttl: Duration::from_secs(86_400), + ocsp_default_ttl: Duration::from_secs(3600), + ocsp_refresh_before: Duration::from_secs(300), + rim_ttl: Duration::from_secs(86_400), + rim_max_stale: Duration::from_secs(7 * 86_400), + refresh_interval: Duration::from_secs(600), + max_cache_entries_per_kind: 10, + }) + .await + .unwrap(); + + let nonce = std::process::id() as u8; + assert_eq!( + proxy.handle_ocsp(ocsp_request(nonce)).await.headers()["x-dstack-cache"], + "MISS" + ); + assert_eq!( + proxy + .handle_ocsp(ocsp_request(nonce.wrapping_add(1))) + .await + .headers()["x-dstack-cache"], + "REFRESH" + ); + assert_eq!(requests.load(Ordering::SeqCst), 2); + upstream.abort(); + } + + #[tokio::test] + async fn rim_cache_uses_versioned_id() { + let response = Bytes::from_static(br#"{"id":"NV_GPU_DRIVER_GH100_580.1","rim":"test"}"#); + let (upstream_url, requests, upstream) = + mock_upstream(response.clone(), JSON_CONTENT_TYPE).await; + let cache_dir = tempfile::tempdir().unwrap(); + let proxy = Proxy::new(test_config( + cache_dir.path().to_path_buf(), + upstream_url.clone(), + upstream_url, + )) + .await + .unwrap(); + + let miss = proxy.handle_rim("NV_GPU_DRIVER_GH100_580.1").await; + assert_eq!(miss.headers()["x-dstack-cache"], "MISS"); + assert_eq!( + miss.into_body().collect().await.unwrap().to_bytes(), + response + ); + let hit = proxy.handle_rim("NV_GPU_DRIVER_GH100_580.1").await; + assert_eq!(hit.headers()["x-dstack-cache"], "HIT"); + assert_eq!(requests.load(Ordering::SeqCst), 1); + upstream.abort(); + } + + #[tokio::test] + async fn stale_rim_is_served_when_upstream_is_unreachable() { + let dead = dead_upstream().await; + let cache_dir = tempfile::tempdir().unwrap(); + let proxy = Proxy::new(test_config( + cache_dir.path().to_path_buf(), + dead.clone(), + dead, + )) + .await + .unwrap(); + + let rim_id = "NV_GPU_DRIVER_GH100_580.1"; + let key = hex::encode(Sha256::digest(rim_id.as_bytes())); + let now = now_epoch(); + proxy + .cache + .put( + "rim", + &key, + b"stale-but-signed", + JSON_CONTENT_TYPE, + now - 1000, + now - 10, + Some(EntrySource::RimId(rim_id.to_string())), + ) + .await + .unwrap(); + + let response = proxy.handle_rim(rim_id).await; + assert_eq!(response.headers()["x-dstack-cache"], "STALE"); + assert_eq!( + response.into_body().collect().await.unwrap().to_bytes(), + &b"stale-but-signed"[..] + ); + } + + #[tokio::test] + async fn expired_ocsp_is_never_served() { + let dead = dead_upstream().await; + let cache_dir = tempfile::tempdir().unwrap(); + let proxy = Proxy::new(test_config( + cache_dir.path().to_path_buf(), + dead.clone(), + dead, + )) + .await + .unwrap(); + + let request = ocsp_request(1); + let key = ocsp::request_cache_key(&request).unwrap(); + let now = now_epoch(); + proxy + .cache + .put( + "ocsp", + &key, + b"expired-response", + OCSP_CONTENT_TYPE, + now - 1000, + now - 10, + Some(EntrySource::OcspRequestB64(BASE64.encode(&request))), + ) + .await + .unwrap(); + + let response = proxy.handle_ocsp(request).await; + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + } + + #[tokio::test] + async fn refresher_renews_entries_past_half_life() { + let now = now_epoch(); + let response = ocsp_response(now); + let (upstream_url, requests, upstream) = + mock_upstream(response.clone(), OCSP_CONTENT_TYPE).await; + let cache_dir = tempfile::tempdir().unwrap(); + let proxy = Proxy::new(test_config( + cache_dir.path().to_path_buf(), + upstream_url, + Url::parse("https://rim.attestation.nvidia.com").unwrap(), + )) + .await + .unwrap(); + + let request = ocsp_request(1); + let key = ocsp::request_cache_key(&request).unwrap(); + // Past half-life (age 1000 of a 1010-second lifetime), close to expiry. + proxy + .cache + .put( + "ocsp", + &key, + b"old-response", + OCSP_CONTENT_TYPE, + now - 1000, + now + 10, + Some(EntrySource::OcspRequestB64(BASE64.encode(&request))), + ) + .await + .unwrap(); + + proxy.refresh_once().await; + assert_eq!(requests.load(Ordering::SeqCst), 1); + let entry = proxy.cache_get("ocsp", &key, 0).await.unwrap(); + assert!(entry.expires_at > now + 10); + assert_eq!(entry.body, response); + + // A freshly renewed entry is not refreshed again on the next sweep. + proxy.refresh_once().await; + assert_eq!(requests.load(Ordering::SeqCst), 1); + // Clients now hit the refreshed entry without touching upstream. + let hit = proxy.handle_ocsp(ocsp_request(2)).await; + assert_eq!(hit.headers()["x-dstack-cache"], "HIT"); + assert_eq!(requests.load(Ordering::SeqCst), 1); + upstream.abort(); + } + + #[tokio::test] + async fn refresher_drops_entries_past_usefulness() { + let dead = dead_upstream().await; + let cache_dir = tempfile::tempdir().unwrap(); + let config = test_config(cache_dir.path().to_path_buf(), dead.clone(), dead); + let max_stale = config.rim_max_stale.as_secs() as i64; + let proxy = Proxy::new(config).await.unwrap(); + + let now = now_epoch(); + proxy + .cache + .put( + "rim", + "aaaa", + b"ancient", + JSON_CONTENT_TYPE, + now - max_stale - 1000, + now - max_stale - 100, + Some(EntrySource::RimId("OLD".into())), + ) + .await + .unwrap(); + proxy + .cache + .put( + "ocsp", + "bbbb", + b"expired", + OCSP_CONTENT_TYPE, + now - 1000, + now - 10, + None, + ) + .await + .unwrap(); + + proxy.refresh_once().await; + assert!(proxy.cache.list("rim").await.unwrap().is_empty()); + assert!(proxy.cache.list("ocsp").await.unwrap().is_empty()); + } + + #[tokio::test] + async fn fill_locks_are_released() { + let now = now_epoch(); + let response = ocsp_response(now); + let (upstream_url, _requests, upstream) = + mock_upstream(response.clone(), OCSP_CONTENT_TYPE).await; + let cache_dir = tempfile::tempdir().unwrap(); + let proxy = Proxy::new(test_config( + cache_dir.path().to_path_buf(), + upstream_url.clone(), + upstream_url, + )) + .await + .unwrap(); + + proxy.handle_ocsp(ocsp_request(1)).await; + proxy.handle_rim("NV_GPU_DRIVER_GH100_580.1").await; + assert!(proxy.fills.is_empty()); + upstream.abort(); + } + + #[test] + fn half_life_refresh_decision() { + // 100-second lifetime: refresh from age 50 onwards. + assert!(!should_refresh(1000, 1100, 1049)); + assert!(should_refresh(1000, 1100, 1050)); + // Expired entries still satisfy the predicate; get() decides whether + // they are dropped or (stale RIM) refreshed. + assert!(should_refresh(1000, 1100, 2000)); + // Degenerate lifetimes never refresh. + assert!(!should_refresh(1000, 1000, 2000)); + assert!(!should_refresh(1000, 900, 2000)); + } +} diff --git a/dstack/nvidia-attest-proxy/src/main.rs b/dstack/nvidia-attest-proxy/src/main.rs new file mode 100644 index 000000000..d6073cf7a --- /dev/null +++ b/dstack/nvidia-attest-proxy/src/main.rs @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +use std::{net::SocketAddr, path::PathBuf, time::Duration}; + +use anyhow::{Context, Result}; +use clap::Parser; +use dstack_nvidia_attest_proxy::ProxyConfig; +use tracing_subscriber::EnvFilter; +use url::Url; + +#[derive(Debug, Parser)] +#[command(about = "Persistent NVIDIA OCSP and RIM cache for dstack GPU attestation")] +struct Args { + /// Address on which the HTTP proxy listens. + #[arg( + long, + env = "NVIDIA_ATTEST_PROXY_LISTEN", + default_value = "0.0.0.0:8090" + )] + listen: SocketAddr, + + /// Directory for persistent OCSP and RIM cache entries. + #[arg( + long, + env = "NVIDIA_ATTEST_PROXY_CACHE_DIR", + default_value = "/var/cache/dstack/nvidia-attest-proxy" + )] + cache_dir: PathBuf, + + /// NVIDIA OCSP responder URL. + #[arg( + long, + env = "NVIDIA_ATTEST_PROXY_OCSP_URL", + default_value = "https://ocsp.ndis.nvidia.com" + )] + ocsp_url: Url, + + /// NVIDIA RIM service base URL. + #[arg( + long, + env = "NVIDIA_ATTEST_PROXY_RIM_URL", + default_value = "https://rim.attestation.nvidia.com" + )] + rim_url: Url, + + /// Optional NVIDIA attestation service bearer token. + #[arg(long, env = "NV_ATTESTATION_SERVICE_KEY", hide_env_values = true)] + service_key: Option<String>, + + #[arg(long, env = "NVIDIA_ATTEST_PROXY_REQUEST_TIMEOUT", default_value = "30s", value_parser = parse_duration)] + request_timeout: Duration, + + #[arg(long, env = "NVIDIA_ATTEST_PROXY_CONNECT_TIMEOUT", default_value = "10s", value_parser = parse_duration)] + connect_timeout: Duration, + + /// Maximum time an OCSP response may remain cached. Signed nextUpdate can shorten it. + #[arg(long, env = "NVIDIA_ATTEST_PROXY_OCSP_MAX_TTL", default_value = "24h", value_parser = parse_duration)] + ocsp_max_ttl: Duration, + + /// Cache lifetime for OCSP responses that omit nextUpdate. + #[arg(long, env = "NVIDIA_ATTEST_PROXY_OCSP_DEFAULT_TTL", default_value = "1h", value_parser = parse_duration)] + ocsp_default_ttl: Duration, + + /// Refresh an OCSP response when less than this much validity remains. + #[arg(long, env = "GPU_ATTEST_PROXY_OCSP_REFRESH_BEFORE", default_value = "5m", value_parser = parse_duration)] + ocsp_refresh_before: Duration, + + /// Cache lifetime for version-addressed RIM documents. + #[arg(long, env = "NVIDIA_ATTEST_PROXY_RIM_TTL", default_value = "30d", value_parser = parse_duration)] + rim_ttl: Duration, + + /// How long an expired RIM document may still be served when the upstream + /// cannot provide a fresh copy. RIM documents are signed and + /// version-addressed, so this trades availability, not security. 0 + /// disables stale serving. + #[arg(long, env = "NVIDIA_ATTEST_PROXY_RIM_MAX_STALE", default_value = "7d", value_parser = parse_duration)] + rim_max_stale: Duration, + + /// Interval between background refresh sweeps renewing cache entries past + /// half their lifetime. 0 disables background refresh. + #[arg(long, env = "NVIDIA_ATTEST_PROXY_REFRESH_INTERVAL", default_value = "10m", value_parser = parse_duration)] + refresh_interval: Duration, + + /// Maximum persistent entries in each of the OCSP and RIM caches. + #[arg( + long, + env = "NVIDIA_ATTEST_PROXY_MAX_CACHE_ENTRIES_PER_KIND", + default_value_t = 10_000 + )] + max_cache_entries_per_kind: usize, +} + +fn parse_duration(value: &str) -> Result<Duration, String> { + humantime::parse_duration(value).map_err(|error| error.to_string()) +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), + ) + .init(); + let args = Args::parse(); + dstack_nvidia_attest_proxy::run(ProxyConfig { + listen_addr: args.listen, + cache_dir: args.cache_dir, + ocsp_url: args.ocsp_url, + rim_url: args.rim_url, + service_key: args.service_key, + request_timeout: args.request_timeout, + connect_timeout: args.connect_timeout, + ocsp_max_ttl: args.ocsp_max_ttl, + ocsp_default_ttl: args.ocsp_default_ttl, + ocsp_refresh_before: args.ocsp_refresh_before, + rim_ttl: args.rim_ttl, + rim_max_stale: args.rim_max_stale, + refresh_interval: args.refresh_interval, + max_cache_entries_per_kind: args.max_cache_entries_per_kind, + }) + .await + .context("GPU attestation proxy failed") +} diff --git a/dstack/nvidia-attest-proxy/src/ocsp.rs b/dstack/nvidia-attest-proxy/src/ocsp.rs new file mode 100644 index 000000000..a5570498d --- /dev/null +++ b/dstack/nvidia-attest-proxy/src/ocsp.rs @@ -0,0 +1,357 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::{bail, Context, Result}; +use chrono::{DateTime, NaiveDateTime}; +use sha2::{Digest, Sha256}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ResponseValidity { + this_update: i64, + next_update: Option<i64>, +} + +#[derive(Debug, Clone, Copy)] +struct Tlv<'a> { + tag: u8, + value: &'a [u8], + raw: &'a [u8], +} + +/// Derive a cache key from the request's CertID list, excluding the request +/// extensions where the per-request OCSP nonce is encoded. +pub(crate) fn request_cache_key(request: &[u8]) -> Result<String> { + let outer = parse_single(request, 0x30).context("invalid OCSPRequest")?; + let mut outer_fields = outer.value; + let tbs = take_expected(&mut outer_fields, 0x30).context("missing tbsRequest")?; + + let mut tbs_fields = tbs.value; + while !tbs_fields.is_empty() { + let field = take_tlv(&mut tbs_fields)?; + match field.tag { + // version and requestorName + 0xa0 | 0xa1 => continue, + // requestList is the first untagged SEQUENCE in TBSRequest. + 0x30 => { + validate_request_list(field.value)?; + return Ok(hex::encode(Sha256::digest(field.raw))); + } + _ => bail!("unexpected field in OCSP tbsRequest"), + } + } + bail!("OCSP request has no requestList") +} + +/// Calculate the deadline for a cache entry. The signed `nextUpdate` value is +/// authoritative; `max_ttl_seconds` can only shorten it. NVIDIA's verifier +/// treats a response without nextUpdate as valid for one hour from thisUpdate, +/// so the proxy mirrors that behavior through `default_ttl_seconds`. +pub(crate) fn response_cache_expiry( + response: &[u8], + now: i64, + max_ttl_seconds: u64, + default_ttl_seconds: u64, +) -> Result<i64> { + let validity = response_validity(response)?; + let default_ttl = + i64::try_from(default_ttl_seconds).context("default OCSP TTL is too large")?; + let max_ttl = i64::try_from(max_ttl_seconds).context("maximum OCSP TTL is too large")?; + let mut expires_at = now.checked_add(max_ttl).context("OCSP TTL overflow")?; + for item in validity { + // Do not cache a response which claims to have been produced in the + // future. The guest performs the authoritative freshness check. + if item.this_update > now { + bail!("OCSP response thisUpdate is in the future"); + } + let signed_expiry = item + .next_update + .unwrap_or_else(|| item.this_update.saturating_add(default_ttl)); + expires_at = expires_at.min(signed_expiry); + } + if expires_at <= now { + bail!("OCSP response has expired"); + } + Ok(expires_at) +} + +fn validate_request_list(mut request_list: &[u8]) -> Result<()> { + if request_list.is_empty() { + bail!("OCSP requestList is empty"); + } + while !request_list.is_empty() { + let request = take_expected(&mut request_list, 0x30)?; + let mut fields = request.value; + take_expected(&mut fields, 0x30).context("OCSP Request has no CertID")?; + // A SingleRequest extension is allowed, but no other trailing fields. + if !fields.is_empty() { + take_expected(&mut fields, 0xa0)?; + } + if !fields.is_empty() { + bail!("unexpected field in OCSP Request"); + } + } + Ok(()) +} + +fn response_validity(response: &[u8]) -> Result<Vec<ResponseValidity>> { + let outer = parse_single(response, 0x30).context("invalid OCSPResponse")?; + let mut fields = outer.value; + let status = take_expected(&mut fields, 0x0a).context("missing OCSP responseStatus")?; + if status.value != [0] { + bail!("OCSP response is not successful"); + } + let response_bytes = take_expected(&mut fields, 0xa0).context("missing responseBytes")?; + let response_bytes = parse_single(response_bytes.value, 0x30)?; + let mut fields = response_bytes.value; + take_expected(&mut fields, 0x06).context("missing OCSP responseType")?; + let basic_der = take_expected(&mut fields, 0x04).context("missing BasicOCSPResponse")?; + + let basic = parse_single(basic_der.value, 0x30).context("invalid BasicOCSPResponse")?; + let mut fields = basic.value; + let response_data = take_expected(&mut fields, 0x30).context("missing ResponseData")?; + let mut fields = response_data.value; + + // Optional version. + if peek_tag(fields) == Some(0xa0) { + take_expected(&mut fields, 0xa0)?; + } + // responderID is CHOICE [1] Name / [2] KeyHash. It may be encoded as a + // primitive or constructed context-specific value. + let responder_id = take_tlv(&mut fields).context("missing responderID")?; + let responder_id_tag = responder_id.tag & 0x1f; + if responder_id.tag & 0xc0 != 0x80 || !matches!(responder_id_tag, 1 | 2) { + bail!("invalid OCSP responderID"); + } + take_expected(&mut fields, 0x18).context("missing producedAt")?; + let responses = take_expected(&mut fields, 0x30).context("missing SingleResponses")?; + + let mut single_responses = responses.value; + let mut validity = Vec::new(); + while !single_responses.is_empty() { + let single = take_expected(&mut single_responses, 0x30)?; + let mut fields = single.value; + take_expected(&mut fields, 0x30).context("SingleResponse has no CertID")?; + let cert_status = take_tlv(&mut fields).context("SingleResponse has no certStatus")?; + if cert_status.tag & 0xc0 != 0x80 { + bail!("invalid OCSP certStatus"); + } + let this_update = take_expected(&mut fields, 0x18).context("missing thisUpdate")?; + let this_update = parse_generalized_time(this_update.value)?; + let next_update = if peek_tag(fields) == Some(0xa0) { + let explicit = take_expected(&mut fields, 0xa0)?; + let value = parse_single(explicit.value, 0x18).context("invalid nextUpdate")?; + Some(parse_generalized_time(value.value)?) + } else { + None + }; + // singleExtensions may follow nextUpdate. + if !fields.is_empty() { + take_expected(&mut fields, 0xa1)?; + } + if !fields.is_empty() { + bail!("unexpected field in SingleResponse"); + } + validity.push(ResponseValidity { + this_update, + next_update, + }); + } + if validity.is_empty() { + bail!("OCSP response has no SingleResponse") + } + Ok(validity) +} + +fn parse_generalized_time(value: &[u8]) -> Result<i64> { + let value = std::str::from_utf8(value).context("GeneralizedTime is not ASCII")?; + for format in ["%Y%m%d%H%M%SZ", "%Y%m%d%H%M%S%.fZ"] { + if let Ok(time) = NaiveDateTime::parse_from_str(value, format) { + return Ok(time.and_utc().timestamp()); + } + } + for format in ["%Y%m%d%H%M%S%z", "%Y%m%d%H%M%S%.f%z"] { + if let Ok(time) = DateTime::parse_from_str(value, format) { + return Ok(time.timestamp()); + } + } + bail!("unsupported GeneralizedTime: {value}") +} + +fn parse_single(input: &[u8], expected_tag: u8) -> Result<Tlv<'_>> { + let mut input_ref = input; + let tlv = take_expected(&mut input_ref, expected_tag)?; + if !input_ref.is_empty() { + bail!("trailing DER data"); + } + Ok(tlv) +} + +fn peek_tag(input: &[u8]) -> Option<u8> { + input.first().copied() +} + +fn take_expected<'a>(input: &mut &'a [u8], expected_tag: u8) -> Result<Tlv<'a>> { + let tlv = take_tlv(input)?; + if tlv.tag != expected_tag { + bail!( + "unexpected DER tag: expected {expected_tag:#04x}, got {:#04x}", + tlv.tag + ); + } + Ok(tlv) +} + +fn take_tlv<'a>(input: &mut &'a [u8]) -> Result<Tlv<'a>> { + let original = *input; + if original.len() < 2 { + bail!("truncated DER value"); + } + let tag = original[0]; + if tag & 0x1f == 0x1f { + bail!("high-tag-number DER values are unsupported"); + } + let first_len = original[1]; + let (header_len, value_len) = if first_len & 0x80 == 0 { + (2usize, usize::from(first_len)) + } else { + let count = usize::from(first_len & 0x7f); + if count == 0 || count > std::mem::size_of::<usize>() || original.len() < 2 + count { + bail!("invalid DER length"); + } + if original[2] == 0 { + bail!("non-canonical DER length"); + } + let mut value_len = 0usize; + for byte in &original[2..2 + count] { + value_len = value_len + .checked_mul(256) + .and_then(|value| value.checked_add(usize::from(*byte))) + .context("DER length overflow")?; + } + if value_len < 128 { + bail!("non-canonical DER length"); + } + (2 + count, value_len) + }; + let total_len = header_len + .checked_add(value_len) + .context("DER value length overflow")?; + if original.len() < total_len { + bail!("truncated DER value"); + } + let raw = &original[..total_len]; + let value = &raw[header_len..]; + *input = &original[total_len..]; + Ok(Tlv { tag, value, raw }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn der(tag: u8, value: &[u8]) -> Vec<u8> { + let mut result = vec![tag]; + if value.len() < 128 { + result.push(value.len() as u8); + } else { + let bytes = value.len().to_be_bytes(); + let first = bytes.iter().position(|byte| *byte != 0).unwrap(); + result.push(0x80 | (bytes.len() - first) as u8); + result.extend_from_slice(&bytes[first..]); + } + result.extend_from_slice(value); + result + } + + fn sequence(parts: &[Vec<u8>]) -> Vec<u8> { + der(0x30, &parts.concat()) + } + + fn request(nonce: u8) -> Vec<u8> { + let algorithm = sequence(&[der(0x06, &[0x2b, 0x0e, 0x03, 0x02, 0x1a])]); + let cert_id = sequence(&[ + algorithm, + der(0x04, &[1; 20]), + der(0x04, &[2; 20]), + der(0x02, &[3]), + ]); + let request = sequence(&[cert_id]); + let list = sequence(&[request]); + // The exact extension contents do not matter to the cache key. They + // model requestExtensions [2], where the OCSP nonce lives. + let extensions = der(0xa2, &sequence(&[der(0x04, &[nonce; 16])])); + sequence(&[sequence(&[list, extensions])]) + } + + fn response(this_update: &str, next_update: Option<&str>) -> Vec<u8> { + let cert_id = sequence(&[der(0x02, &[1])]); + let mut single = vec![cert_id, der(0x80, &[]), der(0x18, this_update.as_bytes())]; + if let Some(next_update) = next_update { + single.push(der(0xa0, &der(0x18, next_update.as_bytes()))); + } + let responses = sequence(&[sequence(&single)]); + let response_data = sequence(&[ + der(0x82, &[7; 20]), + der(0x18, b"20260716120000Z"), + responses, + ]); + let basic = sequence(&[ + response_data, + sequence(&[der(0x06, &[0x2a, 0x03])]), + der(0x03, &[0]), + ]); + let response_bytes = sequence(&[ + der( + 0x06, + &[0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x01, 0x01], + ), + der(0x04, &basic), + ]); + sequence(&[der(0x0a, &[0]), der(0xa0, &response_bytes)]) + } + + #[test] + fn cache_key_ignores_ocsp_nonce() { + let nonce = std::process::id() as u8; + assert_eq!( + request_cache_key(&request(nonce)).unwrap(), + request_cache_key(&request(nonce.wrapping_add(1))).unwrap() + ); + } + + #[test] + fn cache_key_rejects_malformed_requests() { + assert!(request_cache_key(&[0x30, 0x01, 0]).is_err()); + } + + #[test] + fn expiry_honors_next_update_and_max_ttl() { + let now = 1_784_202_600; // 2026-07-16T11:50:00Z + let response = response("20260716114500Z", Some("20260716123000Z")); + assert_eq!( + response_cache_expiry(&response, now, 86_400, 3_600).unwrap(), + 1_784_205_000 + ); + assert_eq!( + response_cache_expiry(&response, now, 600, 3_600).unwrap(), + now + 600 + ); + } + + #[test] + fn expiry_without_next_update_uses_default_ttl() { + let now = 1_784_202_600; + let response = response("20260716114500Z", None); + assert_eq!( + response_cache_expiry(&response, now, 86_400, 3_600).unwrap(), + 1_784_205_900 + ); + } + + #[test] + fn expired_response_is_not_cacheable() { + let response = response("20260716100000Z", Some("20260716110000Z")); + assert!(response_cache_expiry(&response, 1_784_202_600, 86_400, 3_600).is_err()); + } +} diff --git a/dstack/port-forward/Cargo.toml b/dstack/port-forward/Cargo.toml deleted file mode 100644 index 5422080d5..000000000 --- a/dstack/port-forward/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> -# -# SPDX-License-Identifier: Apache-2.0 - -[package] -name = "dstack-port-forward" -version.workspace = true -edition.workspace = true - -[dependencies] -anyhow.workspace = true -tracing.workspace = true -tokio = { workspace = true, features = ["net", "rt", "io-util", "macros", "sync", "time"] } -tokio-util = { version = "0.7", features = ["rt"] } -nix = { workspace = true, features = ["fs", "net", "zerocopy"] } - -[dev-dependencies] -tokio = { workspace = true, features = ["full"] } diff --git a/dstack/port-forward/src/lib.rs b/dstack/port-forward/src/lib.rs deleted file mode 100644 index beeb87d9a..000000000 --- a/dstack/port-forward/src/lib.rs +++ /dev/null @@ -1,137 +0,0 @@ -// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> -// -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::HashMap; -use std::net::{IpAddr, SocketAddr}; - -use anyhow::{bail, Result}; -use tokio::task::JoinHandle; -use tokio_util::sync::CancellationToken; - -mod tcp; -mod udp; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum Protocol { - Tcp, - Udp, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ForwardRule { - pub protocol: Protocol, - pub listen_addr: IpAddr, - pub listen_port: u16, - pub target_ip: IpAddr, - pub target_port: u16, -} - -impl ForwardRule { - fn listen_sock(&self) -> SocketAddr { - SocketAddr::new(self.listen_addr, self.listen_port) - } - - fn target_sock(&self) -> SocketAddr { - SocketAddr::new(self.target_ip, self.target_port) - } -} - -struct RunningRule { - cancel: CancellationToken, - task: JoinHandle<()>, -} - -/// Manages a dynamic set of port forwarding rules. -/// -/// Rules can be added and removed at runtime. Dropping the service cancels all -/// forwarding tasks. -pub struct ForwardService { - cancel: CancellationToken, - rules: HashMap<ForwardRule, RunningRule>, -} - -impl ForwardService { - pub fn new() -> Self { - Self { - cancel: CancellationToken::new(), - rules: HashMap::new(), - } - } - - /// Add a forwarding rule. Returns error if the rule already exists. - pub fn add_rule(&mut self, rule: ForwardRule) -> Result<()> { - if self.rules.contains_key(&rule) { - bail!("rule already exists: {:?}", rule); - } - - let token = self.cancel.child_token(); - let listen = rule.listen_sock(); - let target = rule.target_sock(); - - let task = match rule.protocol { - Protocol::Tcp => tokio::spawn(tcp::run_tcp_forwarder(listen, target, token.clone())), - Protocol::Udp => tokio::spawn(udp::run_udp_forwarder(listen, target, token.clone())), - }; - - tracing::info!( - "added forwarding rule: {listen} -> {target} ({:?})", - rule.protocol - ); - self.rules.insert( - rule, - RunningRule { - cancel: token, - task, - }, - ); - Ok(()) - } - - /// Remove a forwarding rule and stop its task. - pub async fn remove_rule(&mut self, rule: &ForwardRule) -> Result<()> { - match self.rules.remove(rule) { - Some(running) => { - running.cancel.cancel(); - let _ = running.task.await; - tracing::info!( - "removed forwarding rule: {} -> {} ({:?})", - rule.listen_sock(), - rule.target_sock(), - rule.protocol, - ); - Ok(()) - } - None => bail!("rule not found: {:?}", rule), - } - } - - /// Number of active rules. - pub fn len(&self) -> usize { - self.rules.len() - } - - pub fn is_empty(&self) -> bool { - self.rules.is_empty() - } - - /// Gracefully stop all forwarding and wait for tasks to finish. - pub async fn shutdown(mut self) { - self.cancel.cancel(); - for (_, running) in std::mem::take(&mut self.rules) { - let _ = running.task.await; - } - } -} - -impl Default for ForwardService { - fn default() -> Self { - Self::new() - } -} - -impl Drop for ForwardService { - fn drop(&mut self) { - self.cancel.cancel(); - } -} diff --git a/dstack/port-forward/src/tcp.rs b/dstack/port-forward/src/tcp.rs deleted file mode 100644 index 50878552f..000000000 --- a/dstack/port-forward/src/tcp.rs +++ /dev/null @@ -1,207 +0,0 @@ -// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> -// -// SPDX-License-Identifier: Apache-2.0 - -use std::net::SocketAddr; -use std::os::unix::io::{AsFd, AsRawFd, BorrowedFd, OwnedFd}; - -use tokio::io; -use tokio::net::{TcpListener, TcpStream}; -use tokio_util::sync::CancellationToken; - -/// Run a TCP port forwarder: listen on `listen_addr`, forward to `target`. -pub async fn run_tcp_forwarder( - listen_addr: SocketAddr, - target: SocketAddr, - cancel: CancellationToken, -) { - let listener = match TcpListener::bind(listen_addr).await { - Ok(l) => l, - Err(e) => { - tracing::error!("tcp bind {listen_addr} failed: {e}"); - return; - } - }; - tracing::info!("tcp forwarding {listen_addr} -> {target}"); - - loop { - tokio::select! { - _ = cancel.cancelled() => break, - result = listener.accept() => { - let (client, peer) = match result { - Ok(v) => v, - Err(e) => { - tracing::warn!("tcp accept on {listen_addr}: {e}"); - continue; - } - }; - let cancel = cancel.child_token(); - tokio::spawn(handle_tcp_connection(client, peer, target, cancel)); - } - } - } -} - -async fn handle_tcp_connection( - mut client: TcpStream, - peer: SocketAddr, - target: SocketAddr, - cancel: CancellationToken, -) { - let mut server = match TcpStream::connect(target).await { - Ok(s) => s, - Err(e) => { - tracing::debug!("tcp connect to {target} for {peer}: {e}"); - return; - } - }; - - let _ = client.set_nodelay(true); - let _ = server.set_nodelay(true); - - let result = tokio::select! { - _ = cancel.cancelled() => return, - r = relay(&mut client, &mut server) => r, - }; - - if let Err(e) = result { - tracing::debug!("tcp relay {peer} <-> {target}: {e}"); - } -} - -async fn relay(client: &mut TcpStream, server: &mut TcpStream) -> io::Result<()> { - // Try splice(2) zero-copy first, fall back to userspace copy. - match splice_bidirectional(client, server).await { - Ok(()) => Ok(()), - Err(e) if e.kind() == io::ErrorKind::Unsupported => { - tracing::debug!("splice not supported, falling back to copy_bidirectional"); - io::copy_bidirectional(client, server).await?; - Ok(()) - } - Err(e) => Err(e), - } -} - -/// Zero-copy bidirectional TCP relay using Linux splice(2). -/// -/// When one direction hits EOF, select! drops the other direction. -async fn splice_bidirectional(a: &TcpStream, b: &TcpStream) -> io::Result<()> { - let a_fd = a.as_raw_fd(); - let b_fd = b.as_raw_fd(); - - tokio::select! { - r = splice_one_direction(a, a_fd, b, b_fd) => r, - r = splice_one_direction(b, b_fd, a, a_fd) => r, - } -} - -/// Splice data from src fd to dst fd via an intermediate pipe. -/// -/// Uses `TcpStream::try_io` for proper readiness handling: when splice returns -/// EAGAIN, try_io automatically clears the readiness flag so the next -/// `readable().await` / `writable().await` blocks until the fd is truly ready. -async fn splice_one_direction( - src: &TcpStream, - src_fd: i32, - dst: &TcpStream, - dst_fd: i32, -) -> io::Result<()> { - use nix::fcntl::{splice, SpliceFFlags}; - use nix::unistd::pipe; - - let (pipe_r, pipe_w): (OwnedFd, OwnedFd) = pipe().map_err(io::Error::other)?; - - let flags = SpliceFFlags::SPLICE_F_NONBLOCK | SpliceFFlags::SPLICE_F_MOVE; - let chunk: usize = 65536; - - let src_bfd = unsafe { BorrowedFd::borrow_raw(src_fd) }; - let dst_bfd = unsafe { BorrowedFd::borrow_raw(dst_fd) }; - - loop { - src.readable().await?; - - let n = match src.try_io(io::Interest::READABLE, || { - match splice(src_bfd, None, pipe_w.as_fd(), None, chunk, flags) { - Ok(n) => Ok(n), - Err(nix::errno::Errno::EAGAIN) => Err(io::ErrorKind::WouldBlock.into()), - Err(nix::errno::Errno::EINVAL) => Err(io::Error::new( - io::ErrorKind::Unsupported, - "splice not supported for this fd type", - )), - Err(e) => Err(io::Error::other(e)), - } - }) { - Ok(0) => return Ok(()), - Ok(n) => n, - Err(e) if e.kind() == io::ErrorKind::WouldBlock => continue, - Err(e) => return Err(e), - }; - - let mut written = 0; - while written < n { - dst.writable().await?; - match dst.try_io(io::Interest::WRITABLE, || { - match splice(pipe_r.as_fd(), None, dst_bfd, None, n - written, flags) { - Ok(w) => Ok(w), - Err(nix::errno::Errno::EAGAIN) => Err(io::ErrorKind::WouldBlock.into()), - Err(e) => Err(io::Error::other(e)), - } - }) { - Ok(w) => written += w, - Err(e) if e.kind() == io::ErrorKind::WouldBlock => continue, - Err(e) => return Err(e), - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - - #[tokio::test] - async fn test_tcp_forward_roundtrip() { - // Echo server - let echo = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let echo_addr = echo.local_addr().unwrap(); - tokio::spawn(async move { - loop { - let (mut conn, _) = echo.accept().await.unwrap(); - tokio::spawn(async move { - let mut buf = vec![0u8; 4096]; - loop { - let n = match conn.read(&mut buf).await { - Ok(0) | Err(_) => break, - Ok(n) => n, - }; - if conn.write_all(&buf[..n]).await.is_err() { - break; - } - } - }); - } - }); - - // Start forwarder on a free port - let tmp = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let fwd_addr = tmp.local_addr().unwrap(); - drop(tmp); - - let cancel = CancellationToken::new(); - tokio::spawn(run_tcp_forwarder(fwd_addr, echo_addr, cancel.child_token())); - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - let mut conn = TcpStream::connect(fwd_addr).await.unwrap(); - conn.write_all(b"hello splice").await.unwrap(); - - let mut buf = vec![0u8; 64]; - let n = tokio::time::timeout(std::time::Duration::from_secs(2), conn.read(&mut buf)) - .await - .unwrap() - .unwrap(); - assert_eq!(&buf[..n], b"hello splice"); - - cancel.cancel(); - } -} diff --git a/dstack/port-forward/src/udp.rs b/dstack/port-forward/src/udp.rs deleted file mode 100644 index 78b7af5bd..000000000 --- a/dstack/port-forward/src/udp.rs +++ /dev/null @@ -1,201 +0,0 @@ -// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> -// -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::HashMap; -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use tokio::net::UdpSocket; -use tokio::sync::Mutex; -use tokio_util::sync::CancellationToken; - -const UDP_IDLE_TIMEOUT: Duration = Duration::from_secs(60); -const UDP_BUF_SIZE: usize = 65535; -const CLEANUP_INTERVAL: Duration = Duration::from_secs(10); - -struct ClientState { - /// Ephemeral socket for communicating with the target on behalf of this client. - socket: Arc<UdpSocket>, - last_active: Instant, - /// Cancel token for the return-path task. - _cancel: CancellationToken, -} - -/// Run a UDP port forwarder: listen on `listen_addr`, forward to `target`. -pub async fn run_udp_forwarder( - listen_addr: SocketAddr, - target: SocketAddr, - cancel: CancellationToken, -) { - let host_socket = match UdpSocket::bind(listen_addr).await { - Ok(s) => s, - Err(e) => { - tracing::error!("udp bind {listen_addr} failed: {e}"); - return; - } - }; - tracing::info!("udp forwarding {listen_addr} -> {target}"); - - let host_socket = Arc::new(host_socket); - let clients: Arc<Mutex<HashMap<SocketAddr, ClientState>>> = - Arc::new(Mutex::new(HashMap::new())); - - // Periodic cleanup of idle client entries - let cleanup_clients = clients.clone(); - let cleanup_cancel = cancel.child_token(); - tokio::spawn(async move { - loop { - tokio::select! { - _ = cleanup_cancel.cancelled() => break, - _ = tokio::time::sleep(CLEANUP_INTERVAL) => { - let mut map = cleanup_clients.lock().await; - let now = Instant::now(); - map.retain(|addr, entry| { - let alive = now.duration_since(entry.last_active) < UDP_IDLE_TIMEOUT; - if !alive { - tracing::debug!("udp client {addr} idle timeout"); - } - alive - // Dropping ClientState cancels the return-path task via _cancel - }); - } - } - } - }); - - let mut buf = vec![0u8; UDP_BUF_SIZE]; - - loop { - tokio::select! { - _ = cancel.cancelled() => break, - result = host_socket.recv_from(&mut buf) => { - let (n, client_addr) = match result { - Ok(v) => v, - Err(e) => { - tracing::warn!("udp recv on {listen_addr}: {e}"); - continue; - } - }; - - let data = &buf[..n]; - let mut map = clients.lock().await; - - // Get or create per-client ephemeral socket - let entry = match map.entry(client_addr) { - std::collections::hash_map::Entry::Occupied(e) => e.into_mut(), - std::collections::hash_map::Entry::Vacant(e) => { - let socket = match (|| -> anyhow::Result<Arc<UdpSocket>> { - let std_sock = std::net::UdpSocket::bind("0.0.0.0:0")?; - std_sock.set_nonblocking(true)?; - Ok(Arc::new(UdpSocket::from_std(std_sock)?)) - })() { - Ok(s) => s, - Err(e) => { - tracing::warn!("udp ephemeral socket for {client_addr}: {e}"); - continue; - } - }; - - let return_cancel = cancel.child_token(); - tokio::spawn(udp_return_path( - host_socket.clone(), - socket.clone(), - client_addr, - return_cancel.child_token(), - )); - - e.insert(ClientState { - socket, - last_active: Instant::now(), - _cancel: return_cancel, - }) - } - }; - entry.last_active = Instant::now(); - - // Forward client data to target - if let Err(e) = entry.socket.send_to(data, target).await { - tracing::debug!("udp send to {target} for {client_addr}: {e}"); - } - } - } - } -} - -/// Return path: read from the per-client ephemeral socket, send back to the -/// original client via the host socket. -async fn udp_return_path( - host_socket: Arc<UdpSocket>, - client_socket: Arc<UdpSocket>, - client_addr: SocketAddr, - cancel: CancellationToken, -) { - let mut buf = vec![0u8; UDP_BUF_SIZE]; - - loop { - tokio::select! { - _ = cancel.cancelled() => break, - result = client_socket.recv_from(&mut buf) => { - let (n, _from) = match result { - Ok(v) => v, - Err(e) => { - tracing::debug!("udp return recv for {client_addr}: {e}"); - break; - } - }; - if let Err(e) = host_socket.send_to(&buf[..n], client_addr).await { - tracing::debug!("udp return send to {client_addr}: {e}"); - } - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_udp_forward_roundtrip() { - // UDP echo server - let echo = UdpSocket::bind("127.0.0.1:0").await.unwrap(); - let echo_addr = echo.local_addr().unwrap(); - tokio::spawn(async move { - let mut buf = vec![0u8; UDP_BUF_SIZE]; - loop { - match echo.recv_from(&mut buf).await { - Ok((n, from)) => { - let _ = echo.send_to(&buf[..n], from).await; - } - Err(_) => break, - } - } - }); - - // Start forwarder - let cancel = CancellationToken::new(); - // Bind to get a free port, then release and let forwarder bind - let tmp = UdpSocket::bind("127.0.0.1:0").await.unwrap(); - let fwd_addr = tmp.local_addr().unwrap(); - drop(tmp); - - let token = cancel.child_token(); - tokio::spawn(run_udp_forwarder(fwd_addr, echo_addr, token)); - tokio::time::sleep(Duration::from_millis(50)).await; - - // Send through forwarder - let client = UdpSocket::bind("127.0.0.1:0").await.unwrap(); - client.send_to(b"hello udp", fwd_addr).await.unwrap(); - - let mut buf = vec![0u8; 64]; - let (n, _) = tokio::time::timeout(Duration::from_secs(2), client.recv_from(&mut buf)) - .await - .unwrap() - .unwrap(); - assert_eq!(&buf[..n], b"hello udp"); - - cancel.cancel(); - } -} diff --git a/dstack/ra-rpc/Cargo.toml b/dstack/ra-rpc/Cargo.toml index 0fa6a67a6..a3f4537bc 100644 --- a/dstack/ra-rpc/Cargo.toml +++ b/dstack/ra-rpc/Cargo.toml @@ -13,6 +13,7 @@ license.workspace = true anyhow.workspace = true prpc.workspace = true rocket = { workspace = true, features = ["mtls"], optional = true } +rustls = { workspace = true, optional = true } serde_json.workspace = true tracing.workspace = true reqwest = { workspace = true, default-features = false, features = ["rustls", "charset"], optional = true } @@ -25,8 +26,15 @@ x509-parser.workspace = true prost-types = { workspace = true, optional = true } or-panic.workspace = true +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +reqwest = { workspace = true, default-features = false, features = [ + "rustls", + "charset", +] } + [features] default = ["rocket", "client"] -rocket = ["dep:rocket", "dep:rocket-vsock-listener"] +rocket = ["dep:rocket", "dep:rocket-vsock-listener", "dep:rustls"] client = ["reqwest"] openapi = ["dep:prost-types"] diff --git a/dstack/ra-rpc/src/client.rs b/dstack/ra-rpc/src/client.rs index 07d875363..38b2124eb 100644 --- a/dstack/ra-rpc/src/client.rs +++ b/dstack/ra-rpc/src/client.rs @@ -2,14 +2,17 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::time::Duration; +use std::{sync::Arc, time::Duration}; use anyhow::{bail, Context, Result}; use prpc::{ client::{Error, RequestClient}, Message, }; -use ra_tls::{attestation::VerifiedAttestation, traits::CertExt}; +use ra_tls::{ + attestation::{AttestationVerifier, VerifiedAttestation}, + traits::CertExt, +}; use reqwest::{tls::TlsInfo, Certificate, Client, Identity, Response}; use serde::{de::DeserializeOwned, Serialize}; @@ -38,7 +41,7 @@ pub struct RaClientConfig { tls_ca_cert: Option<String>, #[builder(default = true)] tls_built_in_root_certs: bool, - pccs_url: Option<String>, + attestation_verifier: Option<Arc<AttestationVerifier>>, cert_validator: Option<CertValidator>, } @@ -75,9 +78,13 @@ impl RaClientConfig { builder = builder.tls_certs_only(ca_cert); } let client = builder.build().context("failed to create client")?; + let attestation_verifier = self + .attestation_verifier + .map(Ok) + .unwrap_or_else(|| AttestationVerifier::new_prod(None).map(Arc::new))?; Ok(RaClient { remote_uri: self.remote_uri, - pccs_url: self.pccs_url, + attestation_verifier, client, cert_validator: self.cert_validator, verify_server_attestation: self.verify_server_attestation, @@ -87,7 +94,7 @@ impl RaClientConfig { pub struct RaClient { remote_uri: String, - pccs_url: Option<String>, + attestation_verifier: Arc<AttestationVerifier>, client: Client, cert_validator: Option<CertValidator>, verify_server_attestation: bool, @@ -107,7 +114,7 @@ impl RaClient { remote_uri: String, cert_pem: String, key_pem: String, - pccs_url: Option<String>, + attestation_verifier: Arc<AttestationVerifier>, ) -> Result<Self> { RaClientConfig::builder() .tls_no_check(true) @@ -115,7 +122,7 @@ impl RaClient { .remote_uri(remote_uri) .tls_client_cert(cert_pem) .tls_client_key(key_pem) - .maybe_pccs_url(pccs_url) + .attestation_verifier(attestation_verifier) .build() .into_client() .context("failed to create client") @@ -147,9 +154,11 @@ impl RaClient { Some(attestation) => { let verified_attestation = attestation .into_v1() - .verify_with_ra_pubkey(cert.public_key().raw, self.pccs_url.as_deref()) + .verify_with_ra_pubkey(cert.public_key().raw, &self.attestation_verifier) .await - .context("Failed to verify the attestation report")?; + .context( + "failed to verify the attestation report presented by the server", + )?; Some(verified_attestation) } } @@ -164,6 +173,34 @@ impl RaClient { } } +fn normalize_json_response_body(body: &[u8]) -> &[u8] { + if body.is_empty() { + b"null" + } else { + body + } +} + +#[cfg(test)] +mod response_tests { + use super::normalize_json_response_body; + + #[test] + fn empty_json_response_decodes_as_unit() { + let value: () = serde_json::from_slice(normalize_json_response_body(b"")) + .expect("empty response should decode as unit"); + assert_eq!(value, ()); + } + + #[test] + fn non_empty_json_response_is_unchanged() { + assert_eq!( + normalize_json_response_body(br#"{"value":1}"#), + br#"{"value":1}"# + ); + } +} + impl RequestClient for RaClient { async fn request<T, R>(&self, path: &str, body: T) -> Result<R, Error> where @@ -180,9 +217,17 @@ impl RequestClient for RaClient { .await .context("Failed to send request")?; + // Name the direction explicitly: this validates the *server's* attestation, + // not the client's own quote. Without it the error chain reads as if the + // remote end rejected us. self.try_validate_attestation(&response) .await - .context("Failed to validate attestation")?; + .with_context(|| { + format!( + "failed to validate the server attestation of {}", + self.remote_uri + ) + })?; let status = response.status(); if !status.is_success() { @@ -194,7 +239,8 @@ impl RequestClient for RaClient { .await .context("Failed to read response")? .to_vec(); - let response = serde_json::from_slice(&body).context("Failed to deserialize response")?; + let response = serde_json::from_slice(normalize_json_response_body(&body)) + .context("Failed to deserialize response")?; Ok(response) } } diff --git a/dstack/ra-rpc/src/lib.rs b/dstack/ra-rpc/src/lib.rs index bf1e22080..847563ec5 100644 --- a/dstack/ra-rpc/src/lib.rs +++ b/dstack/ra-rpc/src/lib.rs @@ -13,6 +13,9 @@ use tracing::{error, info}; pub use ra_tls::attestation::{Attestation, VerifiedAttestation}; +#[cfg(feature = "rocket")] +pub mod ratls_client_verifier; + #[cfg(feature = "rocket")] pub mod rocket_helper; @@ -59,6 +62,131 @@ pub struct CallContext<'a, State> { pub remote_app_info: Option<AppInfo>, } +/// An RPC error carrying a status code, so a failure can be reported as +/// something more specific than a generic bad request. +/// +/// `Display` and `source` delegate to the inner error, so attaching a code +/// leaves the rendered error message byte-for-byte unchanged. +#[derive(Debug)] +pub struct CodedError { + code: u16, + inner: anyhow::Error, +} + +impl CodedError { + pub fn code(&self) -> u16 { + self.code + } +} + +impl Display for CodedError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Display::fmt(&self.inner, f) + } +} + +impl std::error::Error for CodedError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.inner.source() + } +} + +/// Attach a status code to an error. +pub trait ErrorExt { + fn with_code(self, code: u16) -> anyhow::Error; +} + +impl<E: Into<anyhow::Error>> ErrorExt for E { + fn with_code(self, code: u16) -> anyhow::Error { + anyhow::Error::new(CodedError { + code, + inner: self.into(), + }) + } +} + +/// Attach a status code to the error of a `Result`. +pub trait ResultExt<T> { + fn with_code(self, code: u16) -> Result<T>; +} + +impl<T, E: Into<anyhow::Error>> ResultExt<T> for std::result::Result<T, E> { + fn with_code(self, code: u16) -> Result<T> { + self.map_err(|e| ErrorExt::with_code(e, code)) + } +} + +/// Fail with a status code, the way `anyhow::bail!` fails without one. +/// +/// `ra_rpc::bail!(501, "...")` is `return Err(anyhow!("...").with_code(501))`. +/// The code is the first argument and is not optional: an error that does not +/// need to choose a status should use `anyhow::bail!` and be reported as +/// [`CODE_BAD_REQUEST`], which is the right default for "the caller sent +/// something wrong". Making the code mandatory keeps the two spellings honest +/// about which failure is which. +/// +/// Everything after the code is passed to `anyhow::anyhow!`, so the format +/// arguments are the ones you already know. +/// +/// ``` +/// fn load(id: &str) -> anyhow::Result<()> { +/// if id.is_empty() { +/// ra_rpc::bail!(404, "no app named {id:?}"); +/// } +/// Ok(()) +/// } +/// +/// let err = load("").unwrap_err(); +/// assert_eq!(ra_rpc::code_of(&err), Some(404)); +/// assert_eq!(err.to_string(), r#"no app named """#); +/// ``` +#[macro_export] +macro_rules! bail { + ($code:expr, $($arg:tt)*) => { + return ::core::result::Result::Err($crate::ErrorExt::with_code( + $crate::__private::anyhow!($($arg)*), + $code, + )) + }; +} + +#[doc(hidden)] +pub mod __private { + pub use anyhow::anyhow; +} + +/// Extract the status code attached to `err`, if any. +/// +/// Walks the whole error chain and returns the outermost code, so a code +/// survives `.context(..)` layers and foreign error types wrapped around it. +/// `anyhow::Error::downcast_ref` alone is not enough: it only follows anyhow's +/// own context chain and stops at the first foreign error type, which would +/// silently drop a code buried under, say, a `thiserror` wrapper. +/// +/// The outermost code wins: the layer closest to the caller has the most +/// context about how the failure should be reported. +pub fn code_of(err: &anyhow::Error) -> Option<u16> { + err.chain() + .filter_map(|cause| cause.downcast_ref::<CodedError>()) + .map(CodedError::code) + .next() +} + +/// Whether `method` is one the service actually implements. +/// +/// `prpc-build` generates the method table and the dispatcher's match arms from +/// the same list, so this answers exactly what the dispatcher would accept. +fn is_known_method<S: PrpcService>(method: &str) -> bool { + S::methods().as_ref().contains(&method) +} + +/// Status code reported for an error that carries no code of its own. +pub const CODE_BAD_REQUEST: u16 = 400; +/// Status code reported when a requested method does not exist. +pub const CODE_NOT_FOUND: u16 = 404; +/// Status code reported when the request body exceeds the configured limit. +pub const CODE_PAYLOAD_TOO_LARGE: u16 = 413; + pub trait RpcCall<State>: Sized { type PrpcService: PrpcService + From<Self> + Send + 'static; @@ -71,31 +199,43 @@ pub trait RpcCall<State>: Sized { is_json: bool, is_query: bool, ) -> (u16, Vec<u8>) { - dispatch_prpc( - method, + let (code, body) = dispatch_prpc( + &method, payload, is_json, is_query, <Self::PrpcService as From<Self>>::from(self), ) - .await + .await; + // The dispatcher reports an unknown method as an ordinary error, so it + // arrives here as a generic bad request. Only that case needs refining, + // so the method table is consulted lazily: a successful call, or a + // failure the service chose a code for, never pays for the lookup. The + // dispatcher's own message is reused verbatim rather than rebuilt here. + if code == CODE_BAD_REQUEST && !is_known_method::<Self::PrpcService>(&method) { + return (CODE_NOT_FOUND, body); + } + (code, body) } } async fn dispatch_prpc( - path: String, + path: &str, data: Vec<u8>, json: bool, query: bool, server: impl PrpcService + Send + 'static, ) -> (u16, Vec<u8>) { info!("dispatching request: {path}"); - let result = server.dispatch_request(&path, data, json, query).await; + let result = server.dispatch_request(path, data, json, query).await; let (code, data) = match result { Ok(data) => (200, data), Err(err) => { error!("rpc error: {err:?}"); - (400, encode_error(json, &err)) + // Services can attach a status code to their error; anything that + // does not is reported as a generic bad request, as before. + let code = code_of(&err).unwrap_or(CODE_BAD_REQUEST); + (code, encode_error(json, &err)) } }; (code, data) @@ -111,3 +251,125 @@ pub fn encode_error(json: bool, error: &impl Display) -> Vec<u8> { encode_message_to_vec(&::prpc::server::ProtoError::new(error)) } } + +#[cfg(test)] +mod coded_error_tests { + use super::{code_of, ErrorExt, ResultExt}; + use anyhow::{anyhow, Context, Error}; + + /// A foreign error type that keeps its source in the chain, like `thiserror`. + #[derive(Debug)] + struct Wrapper(Error); + + impl std::fmt::Display for Wrapper { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "wrapper") + } + } + + impl std::error::Error for Wrapper { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.0.as_ref()) + } + } + + #[test] + fn attaching_a_code_leaves_the_message_unchanged() { + let plain = anyhow!("something went wrong"); + let coded = anyhow!("something went wrong").with_code(403); + assert_eq!(format!("{plain}"), format!("{coded}")); + assert_eq!(format!("{plain:#}"), format!("{coded:#}")); + } + + #[test] + fn an_error_without_a_code_reports_none() { + assert_eq!(code_of(&anyhow!("boom")), None); + } + + #[test] + fn a_code_can_be_attached_to_a_result() { + let err = Err::<(), _>(anyhow!("bad body")) + .with_code(400) + .unwrap_err(); + assert_eq!(code_of(&err), Some(400)); + } + + #[test] + fn a_code_is_found_through_many_context_layers() { + let err = Err::<(), _>(anyhow!("db down").with_code(503)) + .context("a") + .context("b") + .context("c") + .unwrap_err(); + assert_eq!(code_of(&err), Some(503)); + assert_eq!(format!("{err:#}"), "c: b: a: db down"); + } + + #[test] + fn a_code_is_found_under_a_foreign_error_type() { + // This is the case a bare `downcast_ref::<CodedError>()` misses, because + // anyhow stops walking at the first foreign error type. + let err = Error::new(Wrapper(anyhow!("db down").with_code(503))); + assert_eq!(code_of(&err), Some(503)); + } + + #[test] + fn a_code_is_found_under_a_foreign_error_type_plus_context() { + let err = Err::<(), _>(Error::new(Wrapper(anyhow!("db down").with_code(503)))) + .context("outer") + .unwrap_err(); + assert_eq!(code_of(&err), Some(503)); + } + + #[test] + fn the_outermost_code_wins() { + assert_eq!( + code_of(&anyhow!("db down").with_code(503).with_code(404)), + Some(404) + ); + let err = Err::<(), _>(anyhow!("db down").with_code(503)) + .context("mid") + .unwrap_err() + .with_code(404); + assert_eq!(code_of(&err), Some(404)); + } + + /// The macro must produce exactly what the longhand does, including + /// through the `?` and `.context(..)` layers a handler adds on the way + /// out -- that is the whole reason a caller can trust the status it reads. + #[test] + fn bail_attaches_the_code_and_returns() { + fn inner(fail: bool) -> anyhow::Result<u8> { + if fail { + crate::bail!(501, "no {} in this image", "nvattest"); + } + Ok(7) + } + fn outer() -> anyhow::Result<u8> { + inner(true).context("collect evidence") + } + + assert_eq!(inner(false).unwrap(), 7, "the macro must not return early"); + + let err = outer().unwrap_err(); + assert_eq!(code_of(&err), Some(501)); + assert_eq!( + format!("{err:#}"), + "collect evidence: no nvattest in this image" + ); + } + + #[test] + fn a_code_survives_a_question_mark_boundary() { + fn inner() -> anyhow::Result<()> { + Err(anyhow!("db down").with_code(503)) + } + fn outer() -> anyhow::Result<()> { + inner().context("outer failed")?; + Ok(()) + } + let err = outer().unwrap_err(); + assert_eq!(code_of(&err), Some(503)); + assert_eq!(format!("{err:#}"), "outer failed: db down"); + } +} diff --git a/dstack/ra-rpc/src/openapi.rs b/dstack/ra-rpc/src/openapi.rs index 7eb0a0e92..8fe2f1561 100644 --- a/dstack/ra-rpc/src/openapi.rs +++ b/dstack/ra-rpc/src/openapi.rs @@ -402,17 +402,25 @@ fn build_operation( } }), ); - responses.insert( - "400".into(), - json!({ - "description": "RPC error", - "content": { - "application/json": { - "schema": { "$ref": "#/components/schemas/RpcError" } + // A service may report a more specific status than a generic bad request; + // these are the ones the framework itself produces. + for (code, description) in [ + ("400", "RPC error"), + ("404", "No such method"), + ("413", "Request body exceeds the configured limit"), + ] { + responses.insert( + code.into(), + json!({ + "description": description, + "content": { + "application/json": { + "schema": { "$ref": "#/components/schemas/RpcError" } + } } - } - }), - ); + }), + ); + } operation.insert("responses".into(), Value::Object(responses)); Ok(Value::Object(operation)) diff --git a/dstack/ra-rpc/src/ratls_client_verifier.rs b/dstack/ra-rpc/src/ratls_client_verifier.rs new file mode 100644 index 000000000..d3c467808 --- /dev/null +++ b/dstack/ra-rpc/src/ratls_client_verifier.rs @@ -0,0 +1,385 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Verify RA-TLS client certificates by attestation instead of by issuer. +//! +//! Rocket configures mutual TLS through rustls' [`WebPkiClientVerifier`], which pins +//! a CA: a presented client certificate must chain to it or the handshake fails. +//! RA-TLS certificates are self-issued and carry their identity in a TEE quote, so +//! there is no CA to pin. dstack worked around that by having the KMS hand every +//! caller a shared "temp CA" private key purely so the minted certificate would +//! chain somewhere — the CA established nothing, and its key was public by design. +//! +//! [`RaTlsClientVerifier`] replaces the chain check with the check that actually +//! carries meaning: the certificate must carry an attestation. Certificates minted +//! from that temp CA — which is what guests and KMS-to-KMS onboarding still send — +//! are accepted for that attestation rather than for their issuer, so nothing has to +//! change on the client side for them to keep working. Verifying that +//! attestation — and deciding whether the app behind it is authorized — needs +//! network I/O (collateral fetch, auth API) and stays where it already is, in +//! [`crate::rocket_helper`] and the service handlers. Keeping the expensive half +//! out of the handshake also keeps unauthenticated peers from driving it. +//! +//! [`WebPkiClientVerifier`]: https://docs.rs/rustls/latest/rustls/server/struct.WebPkiClientVerifier.html + +use std::sync::Arc; + +use rocket::tls::{CipherSuite, ClientHello, Resolver, ServerConfig, TlsConfig}; +use rocket::{Build, Rocket}; +use rustls::crypto::CryptoProvider; +use rustls::pki_types::{CertificateDer, PrivateKeyDer, UnixTime}; +use rustls::server::danger::{ClientCertVerified, ClientCertVerifier}; +use rustls::server::ServerSessionMemoryCache; +use rustls::{DigitallySignedStruct, DistinguishedName, SignatureScheme}; +use tracing::{debug, info, warn}; +use x509_parser::prelude::FromDer as _; + +/// A rustls client-certificate verifier that accepts any certificate carrying an +/// RA-TLS attestation, whatever signed it. +#[derive(Debug)] +pub struct RaTlsClientVerifier { + provider: Arc<CryptoProvider>, +} + +impl RaTlsClientVerifier { + pub fn new(provider: Arc<CryptoProvider>) -> Self { + Self { provider } + } +} + +impl ClientCertVerifier for RaTlsClientVerifier { + /// Advertise no acceptable CAs, so clients send whatever certificate they hold + /// rather than filtering against a list we do not have. + fn root_hint_subjects(&self) -> &[DistinguishedName] { + &[] + } + + fn offer_client_auth(&self) -> bool { + true + } + + /// Anonymous connections stay allowed, matching the previous + /// `[rpc.tls.mutual] mandatory = false`. Handlers that need an attested caller + /// enforce it themselves; the unauthenticated RPCs (`GetMeta`, `GetTempCaCert`) + /// are reachable without a certificate by design. + fn client_auth_mandatory(&self) -> bool { + false + } + + fn verify_client_cert( + &self, + end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + now: UnixTime, + ) -> Result<ClientCertVerified, rustls::Error> { + let (_, cert) = x509_parser::certificate::X509Certificate::from_der(end_entity.as_ref()) + .map_err(|err| { + rustls::Error::General(format!("failed to parse client certificate: {err}")) + })?; + + // webpki used to enforce the validity window; keep doing so now that it does not. + // Compare as i64: x509 timestamps are signed, and a certificate whose window + // predates the epoch must read as expired rather than as unbounded. + let now_secs = now.as_secs() as i64; + let not_before = cert.validity().not_before.timestamp(); + let not_after = cert.validity().not_after.timestamp(); + if now_secs < not_before { + return Err(rustls::Error::General( + "client certificate is not yet valid".into(), + )); + } + if now_secs > not_after { + return Err(rustls::Error::General( + "client certificate has expired".into(), + )); + } + + // Only presence and decodability are checked here. The quote itself, and its + // binding to this certificate's public key, are verified asynchronously in + // `rocket_helper::handle_prpc_impl` before any handler sees the caller. + match ra_tls::attestation::from_cert(&cert) { + Ok(Some(_)) => Ok(ClientCertVerified::assertion()), + Ok(None) => Err(rustls::Error::General( + "client certificate carries no attestation".into(), + )), + Err(err) => Err(rustls::Error::General(format!( + "failed to decode client attestation: {err}" + ))), + } + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &self.provider.signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &self.provider.signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec<SignatureScheme> { + self.provider + .signature_verification_algorithms + .supported_schemes() + } +} + +/// A Rocket TLS resolver that serves one [`ServerConfig`] wired to +/// [`RaTlsClientVerifier`]. +/// +/// Attach it with `rocket.attach(RaTlsClientAuth::fairing())`. Rustls keeps +/// `ServerConfig::verifier` private, so the verifier cannot be swapped into the config +/// rocket already built: this resolver builds its own from the same `[tls]` section and +/// resolves every connection to it. Rocket still builds its default config, but nothing +/// ever uses it, which is what makes `[tls.mutual]` inert. +/// +/// Building a second config means mirroring what the stock listener sets, and the +/// mirror is deliberate rather than incidental: protocol versions, ALPN, +/// `prefer_server_cipher_order`, the session cache and the ticketer are all set to +/// rocket's values. The single exception is `tls.ciphers`, which selects a crypto +/// provider through a mapping rocket keeps private; a non-default cipher list is +/// warned about at startup rather than silently dropped. +pub struct RaTlsClientAuth { + config: Option<Arc<ServerConfig>>, +} + +#[rocket::async_trait] +impl Resolver for RaTlsClientAuth { + async fn init(rocket: &Rocket<Build>) -> rocket::tls::Result<Self> { + let figment = rocket.figment(); + if figment.find_value("tls").is_err() { + // No TLS configured at all (plain-HTTP dev servers). Resolving to `None` + // lets rocket keep whatever it would have done without us. + warn!("no tls section configured, RA-TLS client auth is inactive"); + return Ok(Self { config: None }); + } + // The section exists, so it must parse. Degrading to `None` here would leave + // rocket serving a listener that never asks for a client certificate, and + // every attested RPC would then fail far from the cause. Fail the launch + // instead - including when the section is only half-written, which is the + // case a plain `.ok()` used to swallow. + let tls: TlsConfig = figment.extract_inner("tls")?; + let config = build_server_config(&tls) + .map_err(|err| rocket::figment::Error::from(err.to_string()))?; + info!("RA-TLS client certificate verification enabled"); + Ok(Self { + config: Some(Arc::new(config)), + }) + } + + async fn resolve(&self, _hello: ClientHello<'_>) -> Option<Arc<ServerConfig>> { + self.config.clone() + } +} + +fn build_server_config(tls: &TlsConfig) -> anyhow::Result<ServerConfig> { + use anyhow::Context as _; + use rustls::pki_types::pem::PemObject as _; + + let provider = CryptoProvider::get_default() + .cloned() + .unwrap_or_else(|| Arc::new(rustls::crypto::ring::default_provider())); + + // Read through rocket's own readers so a path and inline PEM bytes are both + // accepted, exactly as the stock listener accepts them. + let mut certs_reader = tls.certs_reader().context("failed to read tls.certs")?; + let certs: Vec<CertificateDer<'static>> = CertificateDer::pem_reader_iter(&mut certs_reader) + .collect::<Result<_, _>>() + .context("failed to parse tls.certs")?; + let mut key_reader = tls.key_reader().context("failed to read tls.key")?; + let key = PrivateKeyDer::from_pem_reader(&mut key_reader).context("failed to parse tls.key")?; + + if tls.ciphers().ne(CipherSuite::DEFAULT_SET) { + // Rocket derives a crypto provider from this list through a private mapping, + // and only when no process-default provider is installed. Rather than + // duplicating that table, say so instead of dropping the setting silently. + warn!("tls.ciphers is not applied by the RA-TLS resolver; the default provider's suites are used"); + } + + let mut config = rustls::ServerConfig::builder_with_provider(provider.clone()) + .with_safe_default_protocol_versions() + .context("failed to select TLS protocol versions")? + .with_client_cert_verifier(Arc::new(RaTlsClientVerifier::new(provider))) + .with_single_cert(certs, key) + .context("failed to load server certificate")?; + // Everything below mirrors rocket's own listener, so replacing the verifier is the + // only observable difference. Dropping the ticketer and the larger session cache + // would quietly change resumption behaviour for every existing deployment. + config.ignore_client_order = tls.prefer_server_cipher_order(); + config.session_storage = ServerSessionMemoryCache::new(1024); + config.ticketer = rustls::crypto::ring::Ticketer::new().context("failed to build ticketer")?; + config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()]; + debug!("built RA-TLS server config"); + Ok(config) +} + +#[cfg(test)] +mod tests { + use super::*; + use ra_tls::attestation::{Attestation, AttestationQuote, TdxQuote, VersionedAttestation}; + use ra_tls::cert::CertRequest; + use ra_tls::rcgen::{Certificate, KeyPair, PKCS_ECDSA_P256_SHA256}; + use std::time::{Duration, SystemTime}; + + fn verifier() -> RaTlsClientVerifier { + RaTlsClientVerifier::new(Arc::new(rustls::crypto::ring::default_provider())) + } + + fn now() -> UnixTime { + UnixTime::since_unix_epoch( + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap(), + ) + } + + fn attestation() -> VersionedAttestation { + Attestation { + quote: AttestationQuote::DstackTdx(TdxQuote { + quote: vec![1, 2, 3], + event_log: vec![], + }), + runtime_events: vec![], + report_data: [0u8; 64], + config: "".into(), + report: (), + } + .into_versioned() + } + + fn leaf( + att: Option<&VersionedAttestation>, + not_after: Option<SystemTime>, + issuer: Option<(&Certificate, &KeyPair)>, + ) -> Vec<u8> { + let key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap(); + let req = CertRequest::builder() + .subject("test client") + .key(&key) + .maybe_attestation(att) + .maybe_not_after(not_after) + .usage_client_auth(true) + .build(); + let cert = match issuer { + Some((ca_cert, ca_key)) => req.signed_by(ca_cert, ca_key).unwrap(), + None => req.self_signed().unwrap(), + }; + cert.der().to_vec() + } + + fn verify(der: Vec<u8>) -> Result<ClientCertVerified, rustls::Error> { + verifier().verify_client_cert(&CertificateDer::from(der), &[], now()) + } + + #[test] + fn accepts_self_signed_cert_carrying_attestation() { + let att = attestation(); + assert!(verify(leaf(Some(&att), None, None)).is_ok()); + } + + #[test] + fn accepts_ca_signed_cert() { + // What guests and KMS-to-KMS onboarding send today: a certificate minted from + // the KMS temp CA. That chain is no longer pinned, so it is accepted for the + // attestation it carries instead. + let ca_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap(); + let ca_cert = CertRequest::builder() + .subject("Dstack Client Temp CA") + .key(&ca_key) + .ca_level(0) + .build() + .self_signed() + .unwrap(); + let att = attestation(); + assert!(verify(leaf(Some(&att), None, Some((&ca_cert, &ca_key)))).is_ok()); + } + + #[test] + fn rejects_cert_without_attestation() { + let err = verify(leaf(None, None, None)).unwrap_err(); + assert!( + err.to_string().contains("no attestation"), + "unexpected error: {err}" + ); + } + + #[test] + fn rejects_expired_cert() { + let att = attestation(); + let expired = SystemTime::now() - Duration::from_secs(3600); + let err = verify(leaf(Some(&att), Some(expired), None)).unwrap_err(); + assert!( + err.to_string().contains("expired"), + "unexpected error: {err}" + ); + } + + fn server_tls_config(prefer_server_cipher_order: bool) -> TlsConfig { + let key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap(); + let cert = CertRequest::builder() + .subject("test server") + .key(&key) + .usage_server_auth(true) + .build() + .self_signed() + .unwrap(); + TlsConfig::from_bytes(cert.pem().as_bytes(), key.serialize_pem().as_bytes()) + .with_preferred_server_cipher_order(prefer_server_cipher_order) + } + + /// Building a second `ServerConfig` is only safe if it carries rocket's own + /// settings; anything left at a rustls default is a silent behaviour change for + /// deployments that used the stock listener. + #[test] + fn mirrors_rockets_tls_settings() { + let config = build_server_config(&server_tls_config(true)).unwrap(); + assert!( + config.ignore_client_order, + "tls.prefer_server_cipher_order must reach the config" + ); + assert!( + config.ticketer.enabled(), + "rocket configures a ticketer; dropping it changes resumption" + ); + assert_eq!( + config.alpn_protocols, + vec![b"h2".to_vec(), b"http/1.1".to_vec()] + ); + + let config = build_server_config(&server_tls_config(false)).unwrap(); + assert!(!config.ignore_client_order); + } + + #[test] + fn rejects_cert_that_expired_before_the_epoch() { + // x509 timestamps are signed. A notAfter before 1970 must read as expired, + // not as "no upper bound". + let att = attestation(); + let expired = SystemTime::UNIX_EPOCH - Duration::from_secs(365 * 86400); + let err = verify(leaf(Some(&att), Some(expired), None)).unwrap_err(); + assert!( + err.to_string().contains("expired"), + "unexpected error: {err}" + ); + } +} diff --git a/dstack/ra-rpc/src/rocket_helper.rs b/dstack/ra-rpc/src/rocket_helper.rs index ede119569..82627e398 100644 --- a/dstack/ra-rpc/src/rocket_helper.rs +++ b/dstack/ra-rpc/src/rocket_helper.rs @@ -14,10 +14,11 @@ use crate::openapi::{OpenApiDoc, RenderedDoc}; #[cfg(all(feature = "rocket", feature = "openapi"))] use rocket::response::content::{RawHtml, RawJson}; #[cfg(all(feature = "rocket", feature = "openapi"))] -use std::{borrow::Cow, sync::Arc}; +use std::borrow::Cow; +use std::sync::Arc; use anyhow::{Context, Result}; -use ra_tls::traits::CertExt; +use ra_tls::{attestation::AttestationVerifier, traits::CertExt}; use rocket::listener::unix::UnixStream; use rocket::listener::{Connection, Listener}; use rocket::tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; @@ -181,10 +182,9 @@ fn unix_peer_cred(stream: &UnixStream) -> Option<UnixPeerCred> { }) } -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct QuoteVerifier { - pccs_url: Option<String>, - amd_kds_base_url: Option<String>, + verifier: Arc<AttestationVerifier>, } pub mod deps { @@ -316,26 +316,8 @@ impl<'r> FromRequest<'r> for &'r QuoteVerifier { } impl QuoteVerifier { - pub fn new(pccs_url: Option<String>) -> Self { - Self::new_with_amd_kds_base(pccs_url, None) - } - - pub fn new_with_amd_kds_base( - pccs_url: Option<String>, - amd_kds_base_url: Option<String>, - ) -> Self { - Self { - pccs_url, - amd_kds_base_url: amd_kds_base_url - .map(|url| url.trim().to_string()) - .filter(|url| !url.is_empty()), - } - } - - fn configure_amd_kds_base_for_request(&self) { - if let Some(base_url) = &self.amd_kds_base_url { - std::env::set_var("DSTACK_AMD_KDS_BASE_URL", base_url); - } + pub fn new(verifier: Arc<AttestationVerifier>) -> Self { + Self { verifier } } } @@ -343,7 +325,7 @@ async fn read_data(data: Data<'_>, limit: ByteUnit) -> Result<Vec<u8>> { let stream = data.open(limit); let data = stream.into_bytes().await.context("failed to read data")?; if !data.is_complete() { - anyhow::bail!("payload too large"); + crate::bail!(crate::CODE_PAYLOAD_TOO_LARGE, "payload too large"); } Ok(data.into_inner()) } @@ -402,9 +384,12 @@ impl<S> PrpcHandler<'_, '_, S> { Err(err) => { warn!("error handling prpc: {err:?}"); let body = encode_error(json, &err); + let status = crate::code_of(&err) + .and_then(Status::from_code) + .unwrap_or(Status::BadRequest); RpcResponse { is_json: json, - status: Status::BadRequest, + status, body, } } @@ -459,21 +444,6 @@ mod tests { use rocket::tokio; use std::time::{SystemTime, UNIX_EPOCH}; - #[test] - fn quote_verifier_carries_trimmed_amd_kds_base_url() { - let verifier = QuoteVerifier::new_with_amd_kds_base( - None, - Some(" https://mirror.example.com/vcek/v1/ ".to_string()), - ); - assert_eq!( - verifier.amd_kds_base_url.as_deref(), - Some("https://mirror.example.com/vcek/v1/") - ); - - let verifier = QuoteVerifier::new_with_amd_kds_base(None, Some(" ".to_string())); - assert!(verifier.amd_kds_base_url.is_none()); - } - #[test] fn custom_unix_endpoint_maps_to_remote_endpoint() { let endpoint = Endpoint::new(UnixPeerEndpoint { @@ -567,7 +537,6 @@ pub async fn handle_prpc_impl<S, Call: RpcCall<S>>( .flatten(); let attestation = match (request.quote_verifier, attestation) { (Some(quote_verifier), Some(attestation)) => { - quote_verifier.configure_amd_kds_base_for_request(); let pubkey = request .certificate .context("certificate is missing")? @@ -576,7 +545,7 @@ pub async fn handle_prpc_impl<S, Call: RpcCall<S>>( .to_vec(); let verified = attestation .into_v1() - .verify_with_ra_pubkey(&pubkey, quote_verifier.pccs_url.as_deref()) + .verify_with_ra_pubkey(&pubkey, "e_verifier.verifier) .await .context("invalid quote")?; Some(verified) diff --git a/dstack/ra-rpc/tests/ratls_client_auth.rs b/dstack/ra-rpc/tests/ratls_client_auth.rs new file mode 100644 index 000000000..9719c508c --- /dev/null +++ b/dstack/ra-rpc/tests/ratls_client_auth.rs @@ -0,0 +1,241 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! End-to-end check that a Rocket server using [`RaTlsClientAuth`] authenticates +//! clients by the attestation in their certificate rather than by its issuer. +//! +//! The cases that matter: +//! * a certificate minted from a temp CA is accepted (what guests and KMS-to-KMS +//! onboarding send today, and what used to be the only accepted shape); +//! * a self-issued RA-TLS certificate is accepted too, which is what lets clients +//! stop fetching CA material; +//! * a certificate with no attestation is rejected during the handshake; +//! * an anonymous connection still reaches the handler, so the unauthenticated +//! RPCs stay reachable. + +use anyhow::{Context, Result}; +use ra_rpc::ratls_client_verifier::RaTlsClientAuth; +use ra_tls::attestation::{Attestation, AttestationQuote, TdxQuote, VersionedAttestation}; +use ra_tls::cert::CertRequest; +use ra_tls::rcgen::{Certificate, KeyPair, PKCS_ECDSA_P256_SHA256}; +use rocket::tls::Resolver as _; + +struct CertPair { + cert_pem: String, + key_pem: String, +} + +fn attestation() -> VersionedAttestation { + Attestation { + quote: AttestationQuote::DstackTdx(TdxQuote { + quote: vec![1, 2, 3], + event_log: vec![], + }), + runtime_events: vec![], + report_data: [0u8; 64], + config: "".into(), + report: (), + } + .into_versioned() +} + +fn server_cert() -> CertPair { + let key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap(); + let alt_names = vec!["localhost".to_string()]; + let cert = CertRequest::builder() + .subject("test server") + .key(&key) + .alt_names(&alt_names) + .usage_server_auth(true) + .build() + .self_signed() + .unwrap(); + CertPair { + cert_pem: cert.pem(), + key_pem: key.serialize_pem(), + } +} + +fn client_cert(with_attestation: bool, issuer: Option<(&Certificate, &KeyPair)>) -> CertPair { + let key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap(); + let att = attestation(); + let req = CertRequest::builder() + .subject("test client") + .key(&key) + .maybe_attestation(with_attestation.then_some(&att)) + .usage_client_auth(true) + .build(); + let cert = match issuer { + Some((ca_cert, ca_key)) => req.signed_by(ca_cert, ca_key).unwrap(), + None => req.self_signed().unwrap(), + }; + CertPair { + cert_pem: cert.pem(), + key_pem: key.serialize_pem(), + } +} + +fn temp_ca() -> (Certificate, KeyPair) { + let key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap(); + let cert = CertRequest::builder() + .subject("Dstack Client Temp CA") + .key(&key) + .ca_level(0) + .build() + .self_signed() + .unwrap(); + (cert, key) +} + +#[rocket::get("/whoami")] +fn whoami(cert: Option<rocket::mtls::Certificate<'_>>) -> String { + match cert { + None => "anonymous".to_string(), + Some(c) => format!("cn={}", c.subject().common_name().unwrap_or("<none>")), + } +} + +/// Drive the server the way a real caller does: an HTTPS client that presents (or +/// withholds) a client certificate. Server-cert checking is off because the test +/// server is self-issued; the client certificate is the subject here. +async fn probe(port: u16, client: Option<&CertPair>) -> Result<String> { + let mut builder = reqwest::Client::builder() + .danger_accept_invalid_certs(true) + .timeout(std::time::Duration::from_secs(10)); + if let Some(pair) = client { + let identity_pem = format!("{}\n{}", pair.cert_pem, pair.key_pem); + builder = builder.identity(reqwest::Identity::from_pem(identity_pem.as_bytes())?); + } + let response = builder + .build()? + .get(format!("https://127.0.0.1:{port}/whoami")) + .send() + .await + .context("request failed")? + .error_for_status()?; + response.text().await.context("failed to read body") +} + +fn free_port() -> u16 { + let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).unwrap(); + listener.local_addr().unwrap().port() +} + +#[tokio::test(flavor = "multi_thread")] +async fn client_certs_are_authenticated_by_attestation_not_issuer() { + let dir = tempdir(); + let server = server_cert(); + let certs_path = dir.join("server.crt"); + let key_path = dir.join("server.key"); + std::fs::write(&certs_path, &server.cert_pem).unwrap(); + std::fs::write(&key_path, &server.key_pem).unwrap(); + + let port = free_port(); + // Note: no `[tls.mutual]` section at all — the resolver supplies the verifier. + let figment = rocket::Config::figment() + .merge(("port", port)) + .merge(("address", "127.0.0.1")) + .merge(("log_level", "off")) + .merge(("shutdown.ctrlc", false)) + .merge(("tls.certs", certs_path.to_str().unwrap())) + .merge(("tls.key", key_path.to_str().unwrap())); + let rocket = rocket::custom(figment) + .attach(RaTlsClientAuth::fairing()) + .mount("/", rocket::routes![whoami]); + let server_task = tokio::spawn(async move { rocket.launch().await }); + + let mut up = false; + for _ in 0..100 { + assert!(!server_task.is_finished(), "rocket exited during launch"); + if tokio::net::TcpStream::connect(("127.0.0.1", port)) + .await + .is_ok() + { + up = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!(up, "server never came up on port {port}"); + + // A self-issued RA-TLS cert is accepted and reaches the handler. No client sends + // one yet; this is the shape the CA pin used to reject. + let self_signed = client_cert(true, None); + assert_eq!( + probe(port, Some(&self_signed)).await.unwrap(), + "cn=test client" + ); + + // A cert minted from the temp CA — what guests and KMS-to-KMS onboarding present + // today — keeps working. + let (ca_cert, ca_key) = temp_ca(); + let ca_signed = client_cert(true, Some((&ca_cert, &ca_key))); + assert_eq!( + probe(port, Some(&ca_signed)).await.unwrap(), + "cn=test client" + ); + + // A cert with no attestation is refused during the handshake. + let no_attestation = client_cert(false, None); + let err = probe(port, Some(&no_attestation)) + .await + .expect_err("cert without attestation must be rejected"); + assert!( + format!("{err:#}").contains("HandshakeFailure"), + "expected the handshake to be refused, got: {err:#}" + ); + + // Anonymous connections still work, so unauthenticated RPCs stay reachable. + assert_eq!(probe(port, None).await.unwrap(), "anonymous"); + + server_task.abort(); +} + +fn tempdir() -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!("ra-rpc-ratls-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +/// A `tls` section that exists but does not parse must stop the launch. Degrading to +/// "no resolver" would leave rocket serving a listener that never asks for a client +/// certificate, so every attested RPC would fail somewhere far from the cause. +#[tokio::test] +async fn a_half_written_tls_section_fails_init() { + let figment = rocket::Config::figment().merge(("tls.certs", "/nonexistent.pem")); + let rocket = rocket::custom(figment); + let err = RaTlsClientAuth::init(&rocket) + .await + .err() + .expect("a tls section without a key must not be ignored"); + assert!( + format!("{err}").contains("key"), + "expected the missing key to be named, got: {err}" + ); +} + +/// No `tls` section at all is the plain-HTTP dev case: init must succeed and stay +/// inactive rather than fail the launch. (`resolve` is unreachable here - rocket only +/// calls it once a handshake is in flight, and there is no TLS listener to have one.) +#[tokio::test] +async fn no_tls_section_still_launches() { + let rocket = rocket::custom(rocket::Config::figment()); + RaTlsClientAuth::init(&rocket) + .await + .expect("a plain-HTTP server must still launch"); +} + +/// Inline PEM bytes are what rocket's own listener accepts alongside paths. The +/// resolver reads through rocket's readers, so both shapes work. +#[tokio::test] +async fn inline_pem_bytes_are_accepted() { + let server = server_cert(); + let figment = rocket::Config::figment() + .merge(("tls.certs", server.cert_pem.as_bytes().to_vec())) + .merge(("tls.key", server.key_pem.as_bytes().to_vec())); + let rocket = rocket::custom(figment); + RaTlsClientAuth::init(&rocket) + .await + .expect("inline PEM must be accepted, as it is by the stock listener"); +} diff --git a/dstack/ra-rpc/tests/status_codes.rs b/dstack/ra-rpc/tests/status_codes.rs new file mode 100644 index 000000000..925ddb221 --- /dev/null +++ b/dstack/ra-rpc/tests/status_codes.rs @@ -0,0 +1,203 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! End-to-end coverage for the status code an RPC failure is reported with. +//! +//! The service is hand-written rather than generated so the test needs no +//! `.proto` and no build script. It mirrors what `prpc-build` emits: a +//! `methods()` table, and a `dispatch_request` whose fallback arm reports an +//! unknown method as an ordinary error. + +use anyhow::{anyhow, Context, Result}; +use ra_rpc::{CallContext, ErrorExt, RpcCall}; +use rocket::local::asynchronous::Client; + +/// A foreign error type that keeps its source in the chain, like `thiserror`. +#[derive(Debug)] +struct ForeignError(anyhow::Error); + +impl std::fmt::Display for ForeignError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "storage layer failed") + } +} + +impl std::error::Error for ForeignError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.0.as_ref()) + } +} + +#[derive(Clone, Default)] +struct AppState; + +struct DemoHandler; + +struct DemoServer(#[allow(dead_code)] DemoHandler); + +impl From<DemoHandler> for DemoServer { + fn from(handler: DemoHandler) -> Self { + Self(handler) + } +} + +impl prpc::server::Service for DemoServer { + type Methods = &'static [&'static str]; + + fn methods() -> Self::Methods { + &[ + "Echo", + "PlainFail", + "Forbidden", + "Unavailable", + "DeeplyBuried", + ] + } + + async fn dispatch_request( + self, + path: &str, + data: impl AsRef<[u8]>, + _json: bool, + _query: bool, + ) -> Result<Vec<u8>, prpc::server::Error> { + match path { + "Echo" => { + // Decoding the payload is what turns a malformed body into a + // generic bad request, exactly as the generated code does. + let value: serde_json::Value = serde_json::from_slice(data.as_ref())?; + Ok(serde_json::to_vec(&value)?) + } + "PlainFail" => Err(anyhow!("something went wrong")), + "Forbidden" => Err(anyhow!("caller is not authorized").with_code(403)), + // The code is attached at the bottom and wrapped in context on the + // way out; it still has to reach the transport. + "Unavailable" => Err(anyhow!("db down").with_code(503)).context("load app"), + // The worst case: a code buried under a foreign error type and + // three context layers. + "DeeplyBuried" => Err(anyhow::Error::new(ForeignError( + anyhow!("no such app").with_code(404), + ))) + .context("lookup app") + .context("handle request") + .context("serve"), + _ => anyhow::bail!("Service not found: {path}"), + } + } +} + +impl RpcCall<AppState> for DemoHandler { + type PrpcService = DemoServer; + + fn construct(_context: CallContext<'_, AppState>) -> Result<Self> { + Ok(DemoHandler) + } +} + +async fn client() -> Client { + let rocket = rocket::build().manage(AppState).mount( + "/", + ra_rpc::prpc_routes!(AppState, DemoHandler, trim: "Demo."), + ); + Client::tracked(rocket) + .await + .expect("failed to build the test client") +} + +async fn post(client: &Client, path: &str, body: &str) -> (u16, String) { + let response = client + .post(format!("/{path}?json")) + .body(body.to_string()) + .dispatch() + .await; + let status = response.status().code; + let text = response.into_string().await.unwrap_or_default(); + (status, text) +} + +#[tokio::test] +async fn a_successful_call_is_reported_as_ok() { + let client = client().await; + let (status, body) = post(&client, "Demo.Echo", r#"{"text":"hello"}"#).await; + assert_eq!(status, 200); + assert_eq!(body, r#"{"text":"hello"}"#); +} + +#[tokio::test] +async fn an_unknown_method_is_reported_as_not_found() { + let client = client().await; + let (status, body) = post(&client, "Demo.NoSuchMethod", "{}").await; + assert_eq!(status, 404); + // The dispatcher's own message is reused rather than rebuilt. + assert!(body.contains("Service not found: NoSuchMethod"), "{body}"); +} + +#[tokio::test] +async fn a_malformed_body_is_reported_as_a_bad_request() { + let client = client().await; + let (status, _) = post(&client, "Demo.Echo", "{not json").await; + assert_eq!(status, 400); +} + +#[tokio::test] +async fn an_error_without_a_code_is_reported_as_a_bad_request() { + let client = client().await; + let (status, body) = post(&client, "Demo.PlainFail", "{}").await; + assert_eq!(status, 400); + assert!(body.contains("something went wrong"), "{body}"); +} + +#[tokio::test] +async fn a_service_can_choose_the_status_code() { + let client = client().await; + let (status, body) = post(&client, "Demo.Forbidden", "{}").await; + assert_eq!(status, 403); + assert!(body.contains("caller is not authorized"), "{body}"); +} + +#[tokio::test] +async fn a_code_survives_a_context_layer() { + let client = client().await; + let (status, body) = post(&client, "Demo.Unavailable", "{}").await; + assert_eq!(status, 503); + // The message reads as if no code had been attached at all. + assert!(body.contains("load app: db down"), "{body}"); + assert!(!body.contains("503"), "{body}"); +} + +#[tokio::test] +async fn a_code_survives_a_foreign_error_type_and_several_context_layers() { + let client = client().await; + let (status, body) = post(&client, "Demo.DeeplyBuried", "{}").await; + assert_eq!(status, 404); + assert!( + body.contains("serve: handle request: lookup app: storage layer failed: no such app"), + "{body}" + ); +} + +#[tokio::test] +async fn a_body_over_the_limit_is_reported_as_payload_too_large() { + // `limit_for_method` looks the limit up by method name. + let figment = rocket::Config::figment().merge(( + "limits", + rocket::data::Limits::default().limit("Echo", rocket::data::ToByteUnit::bytes(8u64)), + )); + let rocket = rocket::custom(figment).manage(AppState).mount( + "/", + ra_rpc::prpc_routes!(AppState, DemoHandler, trim: "Demo."), + ); + let client = Client::tracked(rocket) + .await + .expect("failed to build the test client"); + + let oversized = format!(r#"{{"text":"{}"}}"#, "x".repeat(64)); + let (status, body) = post(&client, "Demo.Echo", &oversized).await; + assert_eq!(status, 413, "{body}"); + assert!(body.contains("payload too large"), "{body}"); + + // A body within the limit still succeeds. + let (status, _) = post(&client, "Demo.Echo", "{}").await; + assert_eq!(status, 200); +} diff --git a/dstack/ra-tls/Cargo.toml b/dstack/ra-tls/Cargo.toml index 3725f3330..236f85ebf 100644 --- a/dstack/ra-tls/Cargo.toml +++ b/dstack/ra-tls/Cargo.toml @@ -17,6 +17,7 @@ elliptic-curve.workspace = true fs-err.workspace = true hex.workspace = true hkdf.workspace = true +k256.workspace = true p256.workspace = true rcgen = { workspace = true, features = ["x509-parser", "pem"] } ring = { workspace = true, features = ["std"] } @@ -46,7 +47,16 @@ errify.workspace = true rmp-serde.workspace = true [features] +simulator = ["dstack-attest/simulator"] quote = ["dstack-attest/quote"] +# A self-signed CA and the leaves it signs, for tests that need a certificate but no +# TEE. Off by default: the keys it mints are test material and must not be reachable +# from a production binary. +test-pki = [] [dev-dependencies] +ed25519-dalek.workspace = true +# `test_pki`'s own tests write PEMs to a scratch dir; the module itself takes a +# caller-supplied path and does not depend on this. +tempfile.workspace = true tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/dstack/ra-tls/src/api_v1.rs b/dstack/ra-tls/src/api_v1.rs new file mode 100644 index 000000000..9cddc2969 --- /dev/null +++ b/dstack/ra-tls/src/api_v1.rs @@ -0,0 +1,438 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Normative constants and encodings for the v1 application-key API. +//! +//! Everything here is wire behaviour, specified in `docs/guest-api-v1.md`. The +//! byte strings these functions build are what a relying party re-derives to +//! check a signature chain, so a change to a constant or a field order changes +//! every key and every claim a deployed agent produces. +//! +//! This lives in `ra-tls` rather than in the guest agent so that the agent, the +//! verifier and the Rust SDK link against one definition. The alternative is +//! each of them transcribing the constants out of the specification prose, +//! which is how cross-implementation crypto drift starts. +//! +//! The module is named for the version of the construction, not for the API +//! that happens to expose it: `ra-tls` hosts versioned crypto, and knowing +//! about a guest API is not its job. + +use crate::kdf::derive_key_with_salt; +use anyhow::{anyhow, bail, Context, Result}; +use k256::ecdsa::SigningKey; +use sha3::{Digest, Keccak256}; + +/// The HKDF salt for every v1 derivation. +/// +/// Distinct from [`crate::kdf::LEGACY_SALT`], which gives v1 its own derivation +/// tree rather than a differently-labelled branch of the old one. Under a +/// shared salt the two surfaces are separated only by their HKDF `info`, and +/// the legacy `info` is the caller's `path` verbatim -- so a caller that passed +/// the v1 `info` byte string as a v0 path would reproduce a v1 key. Different +/// salts close that by construction, whatever either side puts in `info`. +pub const KDF_SALT: &[u8] = b"dstack-guest-v1"; + +/// Context tag bound into every v1 key derivation. +pub const KEY_CONTEXT_TAG: &[u8] = b"dstack-guest-v1-key"; + +/// Context tag bound into every v1 signature-chain key claim. +/// +/// Distinct from [`KEY_CONTEXT_TAG`] so that no derivation input can ever be +/// read as a claim, or the other way round -- the two encodings are otherwise +/// built the same way and would share a prefix. +pub const CLAIM_CONTEXT_TAG: &[u8] = b"dstack-guest-v1-key-claim"; + +/// The key types the v1 API serves. +/// +/// A closed set, matched exhaustively: an algorithm name that is not one of +/// these is an error, never a default. v0 defaulted an empty string to +/// secp256k1 and accepted `k256` as an alias, so a caller could ask for nothing +/// in particular and get a key. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeyAlgorithm { + /// secp256k1, SEC1-compressed public keys. + Secp256k1, + /// Ed25519, RFC 8032 raw public keys. + Ed25519, +} + +impl KeyAlgorithm { + /// Parse a wire algorithm name. There is no default and no alias. + pub fn parse(name: &str) -> Result<Self> { + match name { + "secp256k1" => Ok(Self::Secp256k1), + "ed25519" => Ok(Self::Ed25519), + "" => bail!("algorithm is required, use `secp256k1` or `ed25519`"), + other => bail!("unsupported algorithm `{other}`, use `secp256k1` or `ed25519`"), + } + } + + /// The canonical name, and the exact bytes bound into derivations and + /// claims. Never the string the caller sent. + pub fn name(self) -> &'static str { + match self { + Self::Secp256k1 => "secp256k1", + Self::Ed25519 => "ed25519", + } + } +} + +/// Append one length-prefixed field: a 4-byte big-endian length, then the +/// bytes. +/// +/// Prefixing rather than joining with a delimiter is the whole point. A domain +/// is an arbitrary caller-chosen string -- it may contain `:`, `/`, or NUL -- +/// so any delimiter it could also contain lets two different `(domain, +/// algorithm)` pairs encode to the same byte string and share a key. +pub fn push_length_prefixed(out: &mut Vec<u8>, field: &[u8]) -> Result<()> { + let len = u32::try_from(field.len()).context("field is too long to encode")?; + out.extend_from_slice(&len.to_be_bytes()); + out.extend_from_slice(field); + Ok(()) +} + +/// The HKDF `info` for a v1 application key. +/// +/// `LP(tag) || LP(algorithm) || LP(domain)`, where `LP(x)` is `len(x)` as a +/// 4-byte big-endian integer followed by `x`. +pub fn key_derivation_info(domain: &str, algorithm: KeyAlgorithm) -> Result<Vec<u8>> { + let mut info = Vec::new(); + push_length_prefixed(&mut info, KEY_CONTEXT_TAG)?; + push_length_prefixed(&mut info, algorithm.name().as_bytes())?; + push_length_prefixed(&mut info, domain.as_bytes())?; + Ok(info) +} + +/// Derive the 32 raw bytes of a v1 application key. +/// +/// HKDF-SHA256 over the app root secp256k1 key, under [`KDF_SALT`], with the +/// `info` from [`key_derivation_info`]. +pub fn derive_app_key( + app_root_key: &[u8], + domain: &str, + algorithm: KeyAlgorithm, +) -> Result<[u8; 32]> { + let info = key_derivation_info(domain, algorithm)?; + let derived = derive_key_with_salt(KDF_SALT, app_root_key, &[&info], 32) + .map_err(|_| anyhow!("failed to derive the application key"))?; + derived + .as_slice() + .try_into() + .map_err(|_| anyhow!("derived key has the wrong length")) +} + +/// The claim the app root key signs to vouch for a derived public key. +/// +/// `LP(tag) || LP(algorithm) || LP(domain) || LP(public_key)`, with the public +/// key as raw bytes. +/// +/// Raw bytes, and a length prefix in front of them, are what makes this +/// unforgeable through the v0 surface. v0's claim is +/// `keccak256("{purpose}:{hex(pubkey)}")` over a caller-chosen `purpose`, so +/// the app root key can be made to sign nearly any ASCII string that ends in +/// `:` followed by lowercase hex. This encoding ends in `LP(public_key)`, whose +/// four length bytes are `00 00 00 21` for secp256k1 and `00 00 00 20` for +/// ed25519 and sit inside the region a v0 preimage requires to be hex-only. +/// `0x00` is not a hex character, so no `purpose` reproduces this byte string +/// -- the exclusion is structural, not probabilistic. +pub fn key_claim(algorithm: KeyAlgorithm, domain: &str, public_key: &[u8]) -> Result<Vec<u8>> { + let mut claim = Vec::new(); + push_length_prefixed(&mut claim, CLAIM_CONTEXT_TAG)?; + push_length_prefixed(&mut claim, algorithm.name().as_bytes())?; + push_length_prefixed(&mut claim, domain.as_bytes())?; + push_length_prefixed(&mut claim, public_key)?; + Ok(claim) +} + +/// Sign `message` as `keccak256(message)`, recoverably. +/// +/// Returns the 65-byte `r || s || v` envelope every dstack signature chain link +/// uses: `r` and `s` big-endian and low-S normalised, then the one-byte +/// recovery id. The recovery byte lets a relying party recover the signing +/// public key from the link alone. +/// +/// One definition, because three copies of this five-line envelope had already +/// accumulated and a chain link that disagrees about byte order verifies +/// nowhere. +pub fn sign_recoverable_keccak256(key: &SigningKey, message: &[u8]) -> Result<Vec<u8>> { + let (signature, recovery_id) = key + .sign_digest_recoverable(Keccak256::new_with_prefix(message)) + .context("failed to sign the message")?; + let mut envelope = signature.to_vec(); + envelope.push(recovery_id.to_byte()); + Ok(envelope) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The app root key the committed vectors were generated from. + const TEST_APP_ROOT_KEY: [u8; 32] = [ + 0x1A, 0x2B, 0x3C, 0x4D, 0x5E, 0x6F, 0x7A, 0x8B, 0x9C, 0x0D, 0x1E, 0x2F, 0x3A, 0x4B, 0x5C, + 0x6D, 0x7E, 0x8F, 0x9A, 0x0B, 0x1C, 0x2D, 0x3E, 0x4F, 0x5A, 0x6B, 0x7C, 0x8D, 0x9E, 0x0F, + 0x1A, 0x2B, + ]; + + #[test] + fn rejects_an_empty_or_unknown_algorithm() { + assert_eq!( + KeyAlgorithm::parse("").unwrap_err().to_string(), + "algorithm is required, use `secp256k1` or `ed25519`" + ); + assert_eq!( + KeyAlgorithm::parse("k256").unwrap_err().to_string(), + "unsupported algorithm `k256`, use `secp256k1` or `ed25519`" + ); + assert!(KeyAlgorithm::parse("secp256k1_prehashed").is_err()); + assert!(KeyAlgorithm::parse("rsa").is_err()); + } + + #[test] + fn encodes_derivation_info_with_length_prefixes() { + let info = key_derivation_info("storage-encryption", KeyAlgorithm::Secp256k1).unwrap(); + let expected = [ + &19u32.to_be_bytes()[..], + b"dstack-guest-v1-key", + &9u32.to_be_bytes()[..], + b"secp256k1", + &18u32.to_be_bytes()[..], + b"storage-encryption", + ] + .concat(); + assert_eq!(info, expected); + } + + /// A delimiter-joined encoding would collide here; a length-prefixed one + /// cannot. A domain carrying the delimiter is the shape that breaks + /// `join(":")`. + #[test] + fn no_two_domains_encode_the_same_way() { + let a = key_derivation_info("a\u{0}b", KeyAlgorithm::Ed25519).unwrap(); + let b = key_derivation_info("a", KeyAlgorithm::Ed25519).unwrap(); + let c = key_derivation_info("ab", KeyAlgorithm::Ed25519).unwrap(); + assert_ne!(a, b); + assert_ne!(a, c); + assert_ne!(b, c); + } + + /// Committed vectors, also published in `docs/guest-api-v1.md` so a + /// non-Rust implementation can check itself against them. + /// + /// These bytes are the v1 keys of every deployment whose app root key is + /// `TEST_APP_ROOT_KEY`. A diff here is a change to deployed key material, + /// not a fixture update: fix the derivation, do not update the vector. + #[test] + fn derives_the_committed_key_vectors() { + let vectors = [ + ( + "", + KeyAlgorithm::Secp256k1, + "59f60584ce6fd2a3a31997256db9d77322463fc8a6b1520110401bcb1ee92387", + ), + ( + "", + KeyAlgorithm::Ed25519, + "b023493030669cf22e9cafa6a464d4cf3ae4edfe5474ec796710f21ea011946d", + ), + ( + "storage-encryption", + KeyAlgorithm::Secp256k1, + "5510330f86902ddae38c6d89c93a8408019332c17a429e1abd01c4a28d1544a6", + ), + ( + "storage-encryption", + KeyAlgorithm::Ed25519, + "3c4c3ece12fa99ccb93fc0090877f80e70545fdd971e2ac93d3398c4684538d3", + ), + ( + "a/b/c", + KeyAlgorithm::Secp256k1, + "7f0973449298085d2d36a3b4c4d3243c100ba1981ffa885fe9e9dee883e69538", + ), + // A domain carrying NUL and `:`, the two characters a + // delimiter-joined encoding would choke on. + ( + "k\u{0}:ey", + KeyAlgorithm::Ed25519, + "42da8bf0b479ed125c370e3b91f982735bf08ff592abbd586985affa43ee96a1", + ), + ]; + for (domain, algorithm, expected) in vectors { + let key = derive_app_key(&TEST_APP_ROOT_KEY, domain, algorithm).unwrap(); + assert_eq!( + hex::encode(key), + expected, + "v1 key vector changed for ({domain:?}, {})", + algorithm.name() + ); + } + } + + #[test] + fn the_two_algorithms_never_share_key_material() { + for domain in ["", "storage-encryption", "a/b/c", "\u{0}"] { + assert_ne!( + derive_app_key(&TEST_APP_ROOT_KEY, domain, KeyAlgorithm::Secp256k1).unwrap(), + derive_app_key(&TEST_APP_ROOT_KEY, domain, KeyAlgorithm::Ed25519).unwrap(), + "cross-algorithm key reuse at {domain:?}" + ); + } + } + + /// v0 derived `derive_key(app_root, [path], 32)` and handed the same 32 + /// bytes to both curves. v1 must not land on those bytes when the domain + /// string equals the old path. + #[test] + fn v1_keys_differ_from_v0_keys_for_the_same_name() { + for name in ["", "storage-encryption", "vms"] { + let v0 = crate::kdf::derive_key(&TEST_APP_ROOT_KEY, &[name.as_bytes()], 32).unwrap(); + for algorithm in [KeyAlgorithm::Secp256k1, KeyAlgorithm::Ed25519] { + assert_ne!( + derive_app_key(&TEST_APP_ROOT_KEY, name, algorithm) + .unwrap() + .as_slice(), + v0.as_slice() + ); + } + } + } + + /// The one v0 path that could reach a v1 key under a shared salt: the + /// legacy `info` is the caller's `path` verbatim, so passing the v1 `info` + /// byte string as a v0 path made the two derivations identical. + /// + /// The v1 salt closes it by construction. This is the test that would have + /// failed before the salt changed, so it is the one that keeps it changed. + #[test] + fn a_v0_path_cannot_reproduce_a_v1_key() { + for (domain, algorithm) in [ + ("", KeyAlgorithm::Secp256k1), + ("storage-encryption", KeyAlgorithm::Secp256k1), + ("storage-encryption", KeyAlgorithm::Ed25519), + ] { + let info = key_derivation_info(domain, algorithm).unwrap(); + // The best a v0 caller can do: hand the whole v1 info to `path`. + let v0 = crate::kdf::derive_key(&TEST_APP_ROOT_KEY, &[&info], 32).unwrap(); + assert_ne!( + derive_app_key(&TEST_APP_ROOT_KEY, domain, algorithm) + .unwrap() + .as_slice(), + v0.as_slice(), + "a v0 path reproduced the v1 key for ({domain:?}, {})", + algorithm.name() + ); + } + } + + #[test] + fn the_v1_salt_is_not_the_legacy_salt() { + assert_eq!(KDF_SALT, b"dstack-guest-v1"); + assert_ne!(KDF_SALT, crate::kdf::LEGACY_SALT); + } + + #[test] + fn encodes_the_claim_with_the_raw_public_key() { + let public_key = + hex::decode("03d962450a41748021c8b02787ac36ce642ff0ae25f4c55019eb527e1112cfd764") + .unwrap(); + let claim = key_claim(KeyAlgorithm::Secp256k1, "storage-encryption", &public_key).unwrap(); + let expected = [ + &25u32.to_be_bytes()[..], + b"dstack-guest-v1-key-claim", + &9u32.to_be_bytes()[..], + b"secp256k1", + &18u32.to_be_bytes()[..], + b"storage-encryption", + &33u32.to_be_bytes()[..], + &public_key, + ] + .concat(); + assert_eq!(claim, expected); + } + + /// The forgery a malicious app would attempt: v0's `purpose` is + /// caller-chosen, so it picks one that reproduces the v1 claim's prefix + /// and lets v0 append `":" + hex(pubkey)` itself. + #[test] + fn a_v0_claim_cannot_be_crafted_into_a_v1_claim() { + let public_key = + hex::decode("03d962450a41748021c8b02787ac36ce642ff0ae25f4c55019eb527e1112cfd764") + .unwrap(); + let v1_claim = + key_claim(KeyAlgorithm::Secp256k1, "storage-encryption", &public_key).unwrap(); + + // Best case for the attacker: the v1 claim minus exactly the suffix v0 + // appends on its own, used verbatim as `purpose`. + let hex_pubkey = hex::encode(&public_key); + let appended = format!(":{hex_pubkey}"); + let purpose_len = v1_claim.len().saturating_sub(appended.len()); + let purpose = String::from_utf8_lossy(&v1_claim[..purpose_len]).into_owned(); + let v0_claim = format!("{purpose}:{hex_pubkey}").into_bytes(); + + assert_ne!( + v0_claim, v1_claim, + "a v0 purpose reproduced the v1 claim byte string" + ); + assert_ne!( + Keccak256::digest(&v0_claim), + Keccak256::digest(&v1_claim), + "a v0 claim collided with a v1 claim" + ); + + // And the structural reason, so a future encoding change cannot make + // the assertion above pass by accident: a v0 preimage ends in `:` + // followed by lowercase hex only, while the v1 claim's last 37 bytes + // start with the public key's `00 00 00 21` length prefix. + let tail = &v1_claim[v1_claim.len() - 37..]; + assert_eq!(&tail[..4], &33u32.to_be_bytes()); + assert!( + tail.iter().any(|b| !b.is_ascii_hexdigit()), + "the v1 claim tail is entirely hex, which v0 could reproduce" + ); + } + + #[test] + fn the_two_context_tags_are_not_prefixes_of_one_another_once_encoded() { + let derivation = key_derivation_info("p", KeyAlgorithm::Ed25519).unwrap(); + let claim = key_claim(KeyAlgorithm::Ed25519, "p", &[0u8; 32]).unwrap(); + assert!(!claim.starts_with(&derivation)); + assert!(!derivation.starts_with(&claim)); + } + + /// The claim signature is deterministic: RFC 6979 fixes `k` from the key + /// and the digest, so a committed vector pins the whole chain-link + /// encoding, recovery byte included. + #[test] + fn produces_the_committed_claim_signature_vector() { + let public_key = + hex::decode("03d962450a41748021c8b02787ac36ce642ff0ae25f4c55019eb527e1112cfd764") + .unwrap(); + let claim = key_claim(KeyAlgorithm::Secp256k1, "storage-encryption", &public_key).unwrap(); + let key = SigningKey::from_slice(&TEST_APP_ROOT_KEY).unwrap(); + assert_eq!( + hex::encode(sign_recoverable_keccak256(&key, &claim).unwrap()), + "5b6193729ce7976ec67863f21692d4b98c69832698aae8e001a7d33a6f818b6e\ + 46ca950725b6e90e8ca9bcf394abd03ce264bf9b7eec1e91693247f9dd53c269\ + 01" + ); + } + + #[test] + fn the_chain_link_envelope_recovers_the_signer() { + use k256::ecdsa::{RecoveryId, Signature, VerifyingKey}; + + let key = SigningKey::from_slice(&TEST_APP_ROOT_KEY).unwrap(); + let link = sign_recoverable_keccak256(&key, b"message").unwrap(); + assert_eq!(link.len(), 65); + + let recovered = VerifyingKey::recover_from_digest( + Keccak256::new_with_prefix(b"message"), + &Signature::from_slice(&link[..64]).unwrap(), + RecoveryId::from_byte(link[64]).unwrap(), + ) + .unwrap(); + assert_eq!(&recovered, key.verifying_key()); + } +} diff --git a/dstack/ra-tls/src/attestation.rs b/dstack/ra-tls/src/attestation.rs index d82ce4ff2..b97769b63 100644 --- a/dstack/ra-tls/src/attestation.rs +++ b/dstack/ra-tls/src/attestation.rs @@ -9,22 +9,25 @@ pub use dstack_attest::attestation::*; use crate::{oids, traits::CertExt}; use anyhow::{bail, Context, Result}; -/// Verified RA-TLS certificate evidence. +/// Attestation evidence verified and bound to an RA-TLS certificate key. /// -/// This is the certificate-level endpoint identity proof: the embedded dstack -/// attestation has been cryptographically verified and its `report_data` is -/// bound to the certificate SubjectPublicKeyInfo. -pub struct VerifiedRaTlsCert { - /// DER-encoded SubjectPublicKeyInfo from the verified certificate. +/// This type proves only the embedded attestation and its binding to the +/// certificate SubjectPublicKeyInfo. It does not authenticate the certificate +/// issuer or any other certificate extension. Application identity asserted by +/// a KMS-issued certificate is an issuer claim and must be consumed only after +/// normal certificate-chain verification. +pub struct VerifiedRaTlsAttestation { + /// DER-encoded SubjectPublicKeyInfo bound to the attestation. pub public_key_der: Vec<u8>, /// Verified dstack attestation embedded in the certificate. pub attestation: VerifiedAttestation, - /// Optional app id certificate extension. - pub app_id: Option<Vec<u8>>, - /// Optional app info certificate extension. - pub app_info: Option<AppInfo>, - /// Optional dstack certificate usage extension. - pub special_usage: Option<String>, +} + +impl VerifiedRaTlsAttestation { + /// Decode application identity from the verified attestation evidence. + pub fn decode_app_info(&self, allow_dummy: bool) -> Result<AppInfo> { + self.attestation.decode_app_info(allow_dummy) + } } /// Extract attestation from x509 certificate @@ -41,43 +44,47 @@ pub fn from_der(cert: &[u8]) -> Result<Option<VersionedAttestation>> { /// DER SubjectPublicKeyInfo. That binding prevents an operator-controlled /// network endpoint from reusing valid attestation evidence with a different /// TLS key. -pub async fn verify_der(cert: &[u8], pccs_url: Option<&str>) -> Result<VerifiedRaTlsCert> { +/// +/// This function does not verify a certificate chain and deliberately does not +/// return non-attestation certificate extensions. KMS-issued certificates may +/// omit attestation entirely; authenticate their issuer claims through the +/// certificate chain instead. +pub async fn verify_der( + cert: &[u8], + verifier: &AttestationVerifier, +) -> Result<VerifiedRaTlsAttestation> { let (_, cert) = x509_parser::parse_x509_certificate(cert).context("failed to parse certificate")?; - verify_cert(&cert, pccs_url).await + verify_cert(&cert, verifier).await } /// Verify the RA-TLS attestation embedded in a PEM-encoded X.509 certificate. -pub async fn verify_pem(cert: &[u8], pccs_url: Option<&str>) -> Result<VerifiedRaTlsCert> { +pub async fn verify_pem( + cert: &[u8], + verifier: &AttestationVerifier, +) -> Result<VerifiedRaTlsAttestation> { let (_, pem) = x509_parser::pem::parse_x509_pem(cert).context("failed to parse PEM")?; - verify_der(&pem.contents, pccs_url).await + verify_der(&pem.contents, verifier).await } /// Verify the RA-TLS attestation embedded in a parsed X.509 certificate. async fn verify_cert( cert: &x509_parser::prelude::X509Certificate<'_>, - pccs_url: Option<&str>, -) -> Result<VerifiedRaTlsCert> { + verifier: &AttestationVerifier, +) -> Result<VerifiedRaTlsAttestation> { let attestation = from_cert(cert)?.context("RA-TLS attestation extension missing")?; let public_key_der = cert.tbs_certificate.public_key().raw.to_vec(); if public_key_der.is_empty() { bail!("certificate SubjectPublicKeyInfo is empty"); } - let app_id = cert.get_app_id()?; - let app_info = cert.get_app_info()?; - let special_usage = cert.get_special_usage()?; let attestation = attestation .into_v1() - .verify_with_ra_pubkey(&public_key_der, pccs_url) + .verify_with_ra_pubkey(&public_key_der, verifier) .await .context("RA-TLS attestation verification failed")?; - - Ok(VerifiedRaTlsCert { + Ok(VerifiedRaTlsAttestation { public_key_der, attestation, - app_id, - app_info, - special_usage, }) } @@ -139,7 +146,8 @@ mod tests { .self_signed() .unwrap(); - let result = verify_der(cert.der().as_ref(), None).await; + let verifier = AttestationVerifier::new_prod(None).unwrap(); + let result = verify_der(cert.der().as_ref(), &verifier).await; assert!(result.is_err()); let err = result.err().unwrap(); assert!(err.to_string().contains("attestation extension missing")); @@ -158,7 +166,8 @@ mod tests { .self_signed() .unwrap(); - let result = verify_der(cert.der().as_ref(), None).await; + let verifier = AttestationVerifier::new_prod(None).unwrap(); + let result = verify_der(cert.der().as_ref(), &verifier).await; assert!(result.is_err()); let err = result.err().unwrap(); assert!(format!("{err:#}").contains("report data mismatch")); diff --git a/dstack/ra-tls/src/cert.rs b/dstack/ra-tls/src/cert.rs index df8f4aa05..ed3d71583 100644 --- a/dstack/ra-tls/src/cert.rs +++ b/dstack/ra-tls/src/cert.rs @@ -45,6 +45,15 @@ impl CaCert { /// Instantiate a new CA certificate with a given private key and pem cert. pub fn new(pem_cert: String, pem_key: String) -> Result<Self> { let key = KeyPair::from_pem(&pem_key).context("Failed to parse key")?; + let (_, parsed_pem) = x509_parser::pem::parse_x509_pem(pem_cert.as_bytes()) + .context("Failed to parse cert PEM")?; + let parsed_cert = parsed_pem + .parse_x509() + .context("Failed to parse cert DER")?; + anyhow::ensure!( + parsed_cert.public_key().raw == key.public_key_der(), + "CA certificate does not match private key" + ); let cert = CertificateParams::from_ca_cert_pem(&pem_cert).context("Failed to parse cert")?; // TODO: load the cert from the file directly, blocked by https://github.com/rustls/rcgen/issues/274 @@ -373,8 +382,13 @@ impl<Key> CertRequest<'_, Key> { add_ext(&mut params, PHALA_RATLS_APP_ID, app_id); } if let Some(app_info) = self.app_info { + // Encode as a MessagePack map keyed by field name rather than a positional + // array. Field-name keys let readers tolerate fields they do not know, so + // appending a field to `AppInfo` no longer breaks peers built against an + // older definition. Decoding accepts both forms, so certificates issued by + // older releases (positional) stay readable. let app_info_bytes = - rmp_serde::to_vec(&app_info).context("Failed to serialize app info")?; + rmp_serde::to_vec_named(&app_info).context("failed to serialize app info")?; add_ext(&mut params, PHALA_RATLS_APP_INFO, app_info_bytes); } if let Some(usage) = self.special_usage { @@ -750,4 +764,135 @@ mod tests { let expected = "44706c65617365207369676e20636572743a0c0102030040746573742e6578616d706c652e636f6d000100010000000000040900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"; assert_eq!(actual, expected); } + + /// `AppInfo` travels inside RA-TLS certificates, so its MessagePack encoding is a + /// cross-release wire contract: a certificate issued by one build gets parsed by + /// peers built from another, and certificates outlive the process that issued them. + /// + /// Positional (array) encoding made that contract the *field order and count*, so + /// appending a field broke every peer that had not been rebuilt. Encoding as a map + /// keyed by field name makes it the *field names*, which readers can skip when + /// unknown. These tests pin both directions across the v0.5.11 boundary. + mod app_info_encoding { + use super::*; + use serde::{Deserialize, Serialize}; + use serde_human_bytes as hex_bytes; + + /// The exact `AppInfo` layout shipped in v0.5.6 through v0.5.11: eight fields, + /// no `init_script_hashes`. Stands in for a peer built against those releases. + #[derive(Debug, Serialize, Deserialize)] + struct LegacyAppInfo { + #[serde(with = "hex_bytes")] + app_id: Vec<u8>, + #[serde(with = "hex_bytes")] + compose_hash: Vec<u8>, + #[serde(with = "hex_bytes")] + instance_id: Vec<u8>, + #[serde(with = "hex_bytes")] + device_id: Vec<u8>, + #[serde(with = "hex_bytes")] + mr_system: [u8; 32], + #[serde(with = "hex_bytes")] + mr_aggregated: [u8; 32], + #[serde(with = "hex_bytes")] + os_image_hash: Vec<u8>, + #[serde(with = "hex_bytes")] + key_provider_info: Vec<u8>, + } + + /// A v0.5.11 app-info extension body: `LegacyAppInfo` in positional MessagePack, + /// exactly as `rmp_serde::to_vec` emitted it. Leading `0x98` is a fixarray of 8. + const LEGACY_POSITIONAL_APP_INFO: &str = concat!( + "98c403a1a2a3c402b1b2c402c1c2c401d1c42051515151515151515151515151", + "51515151515151515151515151515151515151c4206262626262626262626262", + "626262626262626262626262626262626262626262c402e1e2c4036b6d73", + ); + + /// True when `bytes` opens with a MessagePack map header of any width. The + /// header widens from fixmap to map16 at 16 entries, so matching on the fixmap + /// range alone would start failing precisely when `AppInfo` grows past 15 + /// fields — the case this encoding exists to support. + fn starts_with_msgpack_map(bytes: &[u8]) -> bool { + matches!(bytes.first().copied(), Some(0x80..=0x8f | 0xde | 0xdf)) + } + + fn sample_app_info() -> AppInfo { + AppInfo { + app_id: vec![0xa1, 0xa2, 0xa3], + compose_hash: vec![0xb1, 0xb2], + instance_id: vec![0xc1, 0xc2], + device_id: vec![0xd1], + mr_system: [0x51; 32], + mr_aggregated: [0x62; 32], + os_image_hash: vec![0xe1, 0xe2], + key_provider_info: b"kms".to_vec(), + init_script_hashes: Some(vec![[0xf1; 32].to_vec(), [0xf2; 32].to_vec()]), + } + } + + /// Old certificate, new reader. Certificates issued before this change carry a + /// positional array and must keep parsing; fields added since default in. + #[test] + fn legacy_positional_app_info_still_decodes() { + let bytes = hex::decode(LEGACY_POSITIONAL_APP_INFO).unwrap(); + let decoded: AppInfo = rmp_serde::from_slice(&bytes).unwrap(); + + assert_eq!(decoded.app_id, vec![0xa1, 0xa2, 0xa3]); + assert_eq!(decoded.compose_hash, vec![0xb1, 0xb2]); + assert_eq!(decoded.instance_id, vec![0xc1, 0xc2]); + assert_eq!(decoded.device_id, vec![0xd1]); + assert_eq!(decoded.mr_system, [0x51; 32]); + assert_eq!(decoded.mr_aggregated, [0x62; 32]); + assert_eq!(decoded.os_image_hash, vec![0xe1, 0xe2]); + assert_eq!(decoded.key_provider_info, b"kms".to_vec()); + assert_eq!( + decoded.init_script_hashes, None, + "a field absent from the legacy layout must decode as unbound, not fail" + ); + } + + /// New certificate, old reader. This is the direction positional encoding broke: + /// a v0.5.11 peer decoding a certificate issued by this build. + #[test] + fn named_app_info_decodes_against_legacy_field_set() { + let encoded = rmp_serde::to_vec_named(&sample_app_info()).unwrap(); + + assert!( + starts_with_msgpack_map(&encoded), + "app info must encode as a MessagePack map, not a positional array" + ); + + let legacy: LegacyAppInfo = rmp_serde::from_slice(&encoded) + .expect("a reader without init_script_hashes must skip it, not fail"); + assert_eq!(legacy.app_id, vec![0xa1, 0xa2, 0xa3]); + assert_eq!(legacy.mr_system, [0x51; 32]); + assert_eq!(legacy.key_provider_info, b"kms".to_vec()); + } + + /// The encoding change must not drop or reshape any field on the way through a + /// real certificate extension. + #[test] + fn app_info_survives_a_certificate_round_trip() { + let key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap(); + let app_info = sample_app_info(); + let cert = CertRequest::builder() + .key(&key) + .subject("test.example.com") + .app_info(&app_info) + .build() + .self_signed() + .unwrap(); + + let decoded = cert.get_app_info().unwrap().expect("app info extension"); + assert_eq!(decoded.app_id, app_info.app_id); + assert_eq!(decoded.compose_hash, app_info.compose_hash); + assert_eq!(decoded.instance_id, app_info.instance_id); + assert_eq!(decoded.device_id, app_info.device_id); + assert_eq!(decoded.mr_system, app_info.mr_system); + assert_eq!(decoded.mr_aggregated, app_info.mr_aggregated); + assert_eq!(decoded.os_image_hash, app_info.os_image_hash); + assert_eq!(decoded.key_provider_info, app_info.key_provider_info); + assert_eq!(decoded.init_script_hashes, app_info.init_script_hashes); + } + } } diff --git a/dstack/ra-tls/src/kdf.rs b/dstack/ra-tls/src/kdf.rs index 2b1aa1145..15303bdd5 100644 --- a/dstack/ra-tls/src/kdf.rs +++ b/dstack/ra-tls/src/kdf.rs @@ -19,13 +19,34 @@ impl KeyType for AnySizeKey { } } -/// Derives a key using HKDF-SHA256. +/// The salt every pre-0.6 derivation uses. +/// +/// Load-bearing: it is baked into every key deployed before the versioned guest +/// API. Never change it. +pub const LEGACY_SALT: &[u8] = b"RATLS"; + +/// Derives a key using HKDF-SHA256 under the legacy [`LEGACY_SALT`]. pub fn derive_key( input_key_material: &[u8], context_data: &[&[u8]], key_size: usize, ) -> Result<Vec<u8>, Unspecified> { - let salt = Salt::new(HKDF_SHA256, b"RATLS"); + derive_key_with_salt(LEGACY_SALT, input_key_material, context_data, key_size) +} + +/// Derives a key using HKDF-SHA256 under an explicit salt. +/// +/// A distinct salt gives a genuinely separate derivation tree: two callers using +/// different salts cannot land on the same key however their `context_data` +/// happens to be built, which a shared salt cannot promise when one caller's +/// context is attacker-chosen. +pub fn derive_key_with_salt( + salt: &[u8], + input_key_material: &[u8], + context_data: &[&[u8]], + key_size: usize, +) -> Result<Vec<u8>, Unspecified> { + let salt = Salt::new(HKDF_SHA256, salt); let pseudo_rand_key: Prk = salt.extract(input_key_material); let output_key_material: Okm<AnySizeKey> = pseudo_rand_key.expand(context_data, AnySizeKey(key_size))?; @@ -102,6 +123,28 @@ mod tests { assert!(key.iter().any(|&x| x != 0)); } + /// The refactor that introduced an explicit salt must not have moved the + /// legacy derivation, which every pre-0.6 key depends on. + #[test] + fn derive_key_still_uses_the_legacy_salt() { + let context = [b"context one".as_ref(), b"context two".as_ref()]; + assert_eq!( + derive_key(b"input key material", &context, 32).unwrap(), + derive_key_with_salt(b"RATLS", b"input key material", &context, 32).unwrap() + ); + } + + /// Two salts, one everything else: the outputs must be unrelated. This is + /// what makes a salt change a real domain separation rather than a rename. + #[test] + fn a_different_salt_gives_a_different_key() { + let context = [b"context one".as_ref()]; + assert_ne!( + derive_key_with_salt(b"RATLS", b"ikm", &context, 32).unwrap(), + derive_key_with_salt(b"dstack-guest-v1", b"ikm", &context, 32).unwrap() + ); + } + #[test] fn test_derive_key256() { let key = derive_key(b"input key material", &[b"context one"], 256).unwrap(); diff --git a/dstack/ra-tls/src/lib.rs b/dstack/ra-tls/src/lib.rs index 70bb712e8..8ea8adf48 100644 --- a/dstack/ra-tls/src/lib.rs +++ b/dstack/ra-tls/src/lib.rs @@ -9,7 +9,10 @@ pub extern crate rcgen; pub mod attestation; +pub mod api_v1; pub mod cert; pub mod kdf; pub mod oids; +#[cfg(feature = "test-pki")] +pub mod test_pki; pub mod traits; diff --git a/dstack/ra-tls/src/test_pki.rs b/dstack/ra-tls/src/test_pki.rs new file mode 100644 index 000000000..6816c204a --- /dev/null +++ b/dstack/ra-tls/src/test_pki.rs @@ -0,0 +1,316 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! Test-only PKI: a self-signed CA and the leaves it signs. +//! +//! Every crate that needs a certificate to drive a TLS test used to build one out of +//! raw `rcgen` -- generate a key, set `IsCa::Ca(BasicConstraints::Unconstrained)`, +//! self-sign, write three PEM files -- which is how the same twenty lines ended up in +//! `gateway`'s sync tests, its HTTPS client tests, and `ra-rpc`'s client-auth tests, +//! each subtly different. +//! +//! Nothing here needs a TEE. `PHALA_RATLS_APP_ID` is an ordinary X.509 extension that +//! [`CertRequest`] writes unconditionally, and a peer check that compares app ids never +//! looks at a quote. For material that *does* carry attestation, use +//! [`generate_ra_cert_with_app_id`](crate::cert::generate_ra_cert_with_app_id), which +//! needs the `quote` feature and a real or simulated platform. +//! +//! Private keys produced here are test material and must never reach a production +//! image, which is why this module is behind the off-by-default `test-pki` feature. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use rcgen::{Certificate, KeyPair}; + +use crate::attestation::AppInfo; +use crate::cert::CertRequest; + +/// A certificate and the key that signs for it. +pub struct TestCertKey { + /// The certificate. + pub cert: Certificate, + /// Its private key. + pub key: KeyPair, +} + +impl TestCertKey { + /// The certificate, PEM-encoded. + pub fn cert_pem(&self) -> String { + self.cert.pem() + } + + /// The private key, PEM-encoded. + pub fn key_pem(&self) -> String { + self.key.serialize_pem() + } + + /// The certificate, DER-encoded. + pub fn cert_der(&self) -> Vec<u8> { + self.cert.der().to_vec() + } + + /// The private key, DER-encoded. + pub fn key_der(&self) -> Vec<u8> { + self.key.serialize_der() + } +} + +/// A self-signed CA that exists to sign test leaves. +/// +/// `ca_level(0)` rather than an unconstrained CA: a test CA has no reason to be allowed +/// to mint intermediates, and a root store that only accepts `CA:TRUE` is satisfied +/// either way. +pub struct TestCa(TestCertKey); + +impl TestCa { + /// Mint a new CA. + pub fn new() -> Result<Self> { + Self::named("dstack Test CA") + } + + /// Mint a new CA with a chosen subject. + pub fn named(subject: &str) -> Result<Self> { + let key = KeyPair::generate().context("failed to generate CA key")?; + let cert = CertRequest::builder() + .subject(subject) + .key(&key) + .ca_level(0) + .build() + .self_signed() + .context("failed to self-sign CA")?; + Ok(Self(TestCertKey { cert, key })) + } + + /// The CA certificate and key. + pub fn cert_key(&self) -> &TestCertKey { + &self.0 + } + + /// The CA certificate, PEM-encoded -- what a client's root store loads. + pub fn cert_pem(&self) -> String { + self.0.cert_pem() + } +} + +/// A leaf to mint, self-signed or signed by a [`TestCa`]. +#[derive(Default)] +pub struct TestCert { + subject: String, + alt_names: Vec<String>, + app_id: Option<Vec<u8>>, + app_info: Option<AppInfo>, + server_auth: bool, + client_auth: bool, +} + +impl TestCert { + /// A leaf with the given subject and no extensions beyond the defaults. + pub fn new(subject: &str) -> Self { + Self { + subject: subject.to_string(), + ..Default::default() + } + } + + /// A leaf valid for `127.0.0.1`, usable for both ends of an mTLS connection. + /// + /// The shape almost every local TLS test wants: Rocket and `rustls` both check the + /// SAN against the address dialled, and a test that reuses one leaf for the server + /// and the client needs both usages. + pub fn localhost() -> Self { + Self::new("localhost") + .alt_name("127.0.0.1") + .server_auth(true) + .client_auth(true) + } + + /// Add a subject alternative name. + pub fn alt_name(mut self, name: &str) -> Self { + self.alt_names.push(name.to_string()); + self + } + + /// Stamp `PHALA_RATLS_APP_ID`, which is what a peer identity check reads. + pub fn app_id(mut self, app_id: &[u8]) -> Self { + self.app_id = Some(app_id.to_vec()); + self + } + + /// Stamp `PHALA_RATLS_APP_INFO`, the fuller identity a KMS-issued certificate + /// carries alongside the app id. + /// + /// Separate from [`Self::app_id`] on purpose, so a test can mint the shape that + /// used to be accepted by a fallback -- app info present, app id absent -- and pin + /// that it no longer is. + pub fn app_info(mut self, app_id: &[u8]) -> Self { + self.app_info = Some(AppInfo { + app_id: app_id.to_vec(), + compose_hash: Vec::new(), + instance_id: Vec::new(), + device_id: Vec::new(), + mr_system: [0u8; 32], + mr_aggregated: [0u8; 32], + os_image_hash: Vec::new(), + key_provider_info: Vec::new(), + init_script_hashes: None, + }); + self + } + + /// Mark the leaf usable for server authentication. + pub fn server_auth(mut self, yes: bool) -> Self { + self.server_auth = yes; + self + } + + /// Mark the leaf usable for client authentication. + pub fn client_auth(mut self, yes: bool) -> Self { + self.client_auth = yes; + self + } + + fn request<'a>(&'a self, key: &'a KeyPair) -> crate::cert::CertRequest<'a, KeyPair> { + CertRequest::builder() + .subject(&self.subject) + .key(key) + .alt_names(&self.alt_names) + .maybe_app_id(self.app_id.as_deref()) + .maybe_app_info(self.app_info.as_ref()) + .usage_server_auth(self.server_auth) + .usage_client_auth(self.client_auth) + .build() + } + + /// Mint the leaf, signed by itself. + pub fn self_signed(self) -> Result<TestCertKey> { + let key = KeyPair::generate().context("failed to generate leaf key")?; + let cert = self + .request(&key) + .self_signed() + .context("failed to self-sign leaf")?; + Ok(TestCertKey { cert, key }) + } + + /// Mint the leaf, signed by `ca`. + pub fn signed_by(self, ca: &TestCa) -> Result<TestCertKey> { + let key = KeyPair::generate().context("failed to generate leaf key")?; + let ca = ca.cert_key(); + let cert = self + .request(&key) + .signed_by(&ca.cert, &ca.key) + .context("failed to sign leaf")?; + Ok(TestCertKey { cert, key }) + } + + /// Mint the leaf DER only, signed by itself. + /// + /// For a check that parses a certificate and reads an extension, where the key is + /// never used to complete a handshake. + pub fn self_signed_der(self) -> Result<Vec<u8>> { + Ok(self.self_signed()?.cert_der()) + } +} + +/// Where [`write_mtls_pki`] put the PEM files. +pub struct TestPkiFiles { + /// The leaf certificate, at `<dir>/node.crt`. + pub cert_path: PathBuf, + /// The leaf private key, at `<dir>/node.key`. + pub key_path: PathBuf, + /// The CA certificate, at `<dir>/ca.crt`. + pub ca_cert_path: PathBuf, + /// The leaf, for a test that also needs to serve with it. + pub leaf: TestCertKey, + /// The CA, for a test that signs a second leaf to play the other end. + pub ca: TestCa, +} + +/// Mint a CA and one leaf it signed, and write all three PEM files under `dir`. +/// +/// The layout clients expect: a config that names a cert, a key and a CA bundle can +/// point straight at the three paths returned. Both the leaf and the CA come back, so a +/// test that needs a second peer -- a different app id, an expired leaf -- can sign one +/// under the same CA rather than starting over. +pub fn write_mtls_pki(dir: &Path, leaf: TestCert) -> Result<TestPkiFiles> { + let ca = TestCa::new()?; + let leaf = leaf.signed_by(&ca)?; + + let cert_path = dir.join("node.crt"); + let key_path = dir.join("node.key"); + let ca_cert_path = dir.join("ca.crt"); + + std::fs::write(&cert_path, leaf.cert_pem()).context("failed to write leaf cert")?; + std::fs::write(&key_path, leaf.key_pem()).context("failed to write leaf key")?; + std::fs::write(&ca_cert_path, ca.cert_pem()).context("failed to write CA cert")?; + + Ok(TestPkiFiles { + cert_path, + key_path, + ca_cert_path, + leaf, + ca, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::traits::CertExt; + + #[test] + fn a_leaf_carries_the_app_id_it_was_stamped_with() { + let leaf = TestCert::localhost() + .app_id(b"an-app-id") + .self_signed() + .expect("leaf"); + assert_eq!( + leaf.cert.get_app_id().expect("read app id"), + Some(b"an-app-id".to_vec()) + ); + } + + #[test] + fn a_leaf_without_an_app_id_reads_back_as_absent() { + let leaf = TestCert::localhost().self_signed().expect("leaf"); + assert_eq!(leaf.cert.get_app_id().expect("read app id"), None); + } + + /// A root store built from the CA only accepts a trust anchor with `CA:TRUE`, so a + /// CA that did not come back as one would fail every handshake rather than an + /// assertion here. + #[test] + fn the_ca_is_a_ca_and_signs_leaves_that_chain_to_it() { + use x509_parser::prelude::FromDer; + + let ca = TestCa::new().expect("ca"); + let leaf = TestCert::localhost().signed_by(&ca).expect("leaf"); + + let ca_der = ca.cert_key().cert_der(); + let (_, parsed_ca) = + x509_parser::certificate::X509Certificate::from_der(&ca_der).expect("parse ca"); + assert!(parsed_ca.is_ca(), "the test CA must be usable as one"); + + let leaf_der = leaf.cert_der(); + let (_, parsed_leaf) = + x509_parser::certificate::X509Certificate::from_der(&leaf_der).expect("parse leaf"); + assert_eq!( + parsed_leaf.issuer(), + parsed_ca.subject(), + "a leaf signed by the CA must name it as issuer" + ); + } + + #[test] + fn the_three_pem_files_land_where_a_client_config_points() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = write_mtls_pki(dir.path(), TestCert::localhost().app_id(b"an-app-id")) + .expect("write pki"); + + for path in [&pki.cert_path, &pki.key_path, &pki.ca_cert_path] { + let pem = std::fs::read_to_string(path).expect("read pem"); + assert!(pem.contains("-----BEGIN"), "{path:?} is not PEM"); + } + } +} diff --git a/dstack/ra-tls/src/traits.rs b/dstack/ra-tls/src/traits.rs index 5cd8e7c58..0916039c1 100644 --- a/dstack/ra-tls/src/traits.rs +++ b/dstack/ra-tls/src/traits.rs @@ -43,8 +43,8 @@ pub trait CertExt { let Some(app_info_bytes) = self.get_extension_bytes(PHALA_RATLS_APP_INFO)? else { return Ok(None); }; - let app_info = - rmp_serde::from_slice(&app_info_bytes).context("Failed to decode app info as json")?; + let app_info = rmp_serde::from_slice(&app_info_bytes) + .context("failed to decode app info as msgpack")?; Ok(app_info) } } diff --git a/dstack/run-tests.sh b/dstack/run-tests.sh index 4fb8dbd94..2e9038117 100755 --- a/dstack/run-tests.sh +++ b/dstack/run-tests.sh @@ -8,76 +8,20 @@ set -Eeuo pipefail CORE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" REPO_ROOT="$(cd "$CORE_DIR/.." && pwd -P)" -SIMULATOR_DIR="$REPO_ROOT/sdk/simulator" -SIMULATOR_LOG="$SIMULATOR_DIR/dstack-simulator.log" -DSTACK_SOCKET="$SIMULATOR_DIR/dstack.sock" -TAPPD_SOCKET="$SIMULATOR_DIR/tappd.sock" -GUEST_SOCKET="$SIMULATOR_DIR/guest.sock" -EXTERNAL_SOCKET="$SIMULATOR_DIR/external.sock" -SIMULATOR_PID="" -cleanup() { - if [[ -n "${SIMULATOR_PID:-}" ]]; then - kill "$SIMULATOR_PID" 2>/dev/null || true - wait "$SIMULATOR_PID" 2>/dev/null || true - fi - rm -f "$DSTACK_SOCKET" "$TAPPD_SOCKET" "$GUEST_SOCKET" "$EXTERNAL_SOCKET" -} +# The third runner sharing `sdk/simulator/`: same binary, same four sockets. +# It used to transcribe the lifecycle instead of sourcing it, and inherited the +# lost-pid bug documented in `simulator_start` -- cleanup killed the wrapper and +# left the simulator holding its binary open, so the next run's `build.sh` hit +# "Text file busy". Sourcing keeps the fix in one place. +# shellcheck source=../sdk/simulator/lifecycle.sh +# shellcheck disable=SC1091 # the hook runs without -x, so it cannot follow this +source "$REPO_ROOT/sdk/simulator/lifecycle.sh" -print_simulator_logs() { - if [[ -f "$SIMULATOR_LOG" ]]; then - echo "Last simulator logs:" - tail -100 "$SIMULATOR_LOG" || true - fi -} +trap 'simulator_print_logs' ERR +trap simulator_stop EXIT INT TERM -wait_for_socket() { - local socket_path="$1" - local name="$2" - - for _ in {1..100}; do - if [[ -S "$socket_path" ]]; then - return 0 - fi - if [[ -n "${SIMULATOR_PID:-}" ]] && ! kill -0 "$SIMULATOR_PID" 2>/dev/null; then - echo "Simulator exited before $name socket became ready." - print_simulator_logs - return 1 - fi - sleep 0.2 - done - - echo "Timed out waiting for $name socket at $socket_path" - print_simulator_logs - return 1 -} - -trap 'print_simulator_logs' ERR -trap cleanup EXIT INT TERM - -rm -f \ - "$DSTACK_SOCKET" \ - "$TAPPD_SOCKET" \ - "$GUEST_SOCKET" \ - "$EXTERNAL_SOCKET" \ - "$SIMULATOR_LOG" -( - cd "$SIMULATOR_DIR" - ./build.sh -) - -( - cd "$SIMULATOR_DIR" - ./dstack-simulator >"$SIMULATOR_LOG" 2>&1 -) & -SIMULATOR_PID=$! -echo "Simulator process (PID: $SIMULATOR_PID) started." - -wait_for_socket "$DSTACK_SOCKET" "dstack" -wait_for_socket "$TAPPD_SOCKET" "tappd" - -export DSTACK_SIMULATOR_ENDPOINT="$DSTACK_SOCKET" -export TAPPD_SIMULATOR_ENDPOINT="$TAPPD_SOCKET" +simulator_start echo "DSTACK_SIMULATOR_ENDPOINT: $DSTACK_SIMULATOR_ENDPOINT" echo "TAPPD_SIMULATOR_ENDPOINT: $TAPPD_SIMULATOR_ENDPOINT" diff --git a/dstack/scripts/bin/dstack-cloud b/dstack/scripts/bin/dstack-cloud index cc3afeaad..b481f8fc9 100755 --- a/dstack/scripts/bin/dstack-cloud +++ b/dstack/scripts/bin/dstack-cloud @@ -585,6 +585,7 @@ class CloudDeploymentManager: kms_urls = services.get("kms_urls") or ["https://kms.tdxlab.dstack.org:12001"] gateway_urls = services.get("gateway_urls") or ["https://gateway.tdxlab.dstack.org:12002"] pccs_url = services.get("pccs_url", "") + nvidia_attestation_proxy_url = services.get("nvidia_attestation_proxy_url") platform = (app.platform or "gcp").lower() if platform == "gcp": @@ -651,12 +652,15 @@ class CloudDeploymentManager: }, } - return { + sys_config = { "kms_urls": kms_urls, "gateway_urls": gateway_urls, "pccs_url": pccs_url, "vm_config": json.dumps(vm_config), } + if nvidia_attestation_proxy_url: + sys_config["nvidia_attestation_proxy_url"] = nvidia_attestation_proxy_url + return sys_config def load_state(self) -> Optional[DeploymentState]: """Load deployment state.""" @@ -1004,6 +1008,7 @@ class CloudDeploymentManager: "kms_urls": ["https://kms.tdxlab.dstack.org:12001"], "gateway_urls": ["https://gateway.tdxlab.dstack.org:12002"], "pccs_url": "", + "nvidia_attestation_proxy_url": "", }, "_comment_images": "System image search paths (for boot_image_tar auto-discovery)", "image_search_paths": [ diff --git a/dstack/scripts/dhcp-notify.sh b/dstack/scripts/dhcp-notify.sh deleted file mode 100755 index 697b3dd13..000000000 --- a/dstack/scripts/dhcp-notify.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> -# SPDX-License-Identifier: Apache-2.0 -# -# DHCP lease notification script for dnsmasq. -# -# Called by dnsmasq via --dhcp-script on lease events. -# Notifies dstack-vmm of the guest's MAC and IP so that port -# forwarding can be established automatically. -# -# Arguments (set by dnsmasq): -# $1 action add | del | old -# $2 mac MAC address of the guest NIC -# $3 ip IPv4 address assigned by DHCP -# $4 hostname (optional) -# -# Configuration: -# VMM_URL Base URL of dstack-vmm (default: http://127.0.0.1:9080) - -ACTION="$1" -MAC="$2" -IP="$3" - -VMM_URL="${VMM_URL:-http://127.0.0.1:9080}" - -logger -t dhcp-notify "action=$ACTION mac=$MAC ip=$IP" - -case "$ACTION" in - add|old) - curl -s -X POST "${VMM_URL}/prpc/ReportDhcpLease" \ - -H 'Content-Type: application/json' \ - -d "{\"mac\":\"$MAC\",\"ip\":\"$IP\"}" \ - || logger -t dhcp-notify "failed to notify VMM" - ;; - del) - # Could clear forwarding on lease expiry; not implemented yet. - ;; -esac diff --git a/dstack/scripts/install.sh b/dstack/scripts/install.sh index af245443a..736c18745 100755 --- a/dstack/scripts/install.sh +++ b/dstack/scripts/install.sh @@ -6,7 +6,7 @@ set -eu DEFAULT_REPO="https://github.com/Dstack-TEE/dstack" -DEFAULT_REF="master" +DEFAULT_REF="next" DEFAULT_PREFIX="/usr/local" usage() { @@ -15,13 +15,13 @@ Install dstackup from source. Usage: dstack/scripts/install.sh [options] - curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/dstack/scripts/install.sh | sh + curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/next/dstack/scripts/install.sh | sh Options: --repo URL Git repository to clone when not run from a checkout. Default: https://github.com/Dstack-TEE/dstack --ref REF Git ref to checkout when cloning or updating DSTACK_SRC. - Default: master + Default: next --src DIR Persistent source checkout to build from. Default: a temporary checkout --prefix DIR Install dstackup under DIR/bin. Use the same DIR with @@ -57,11 +57,19 @@ cleanup() { } trap cleanup EXIT INT TERM +info() { + printf '%s\n' "$*" >&2 +} + +error() { + printf 'error: %s\n' "$*" >&2 +} + while [ "$#" -gt 0 ]; do case "$1" in --repo) if [ "$#" -lt 2 ]; then - echo "error: --repo requires a URL" >&2 + error "--repo requires a URL" exit 1 fi repo=$2 @@ -69,7 +77,7 @@ while [ "$#" -gt 0 ]; do ;; --ref) if [ "$#" -lt 2 ]; then - echo "error: --ref requires a ref" >&2 + error "--ref requires a ref" exit 1 fi ref=$2 @@ -77,7 +85,7 @@ while [ "$#" -gt 0 ]; do ;; --src) if [ "$#" -lt 2 ]; then - echo "error: --src requires a directory" >&2 + error "--src requires a directory" exit 1 fi src=$2 @@ -85,7 +93,7 @@ while [ "$#" -gt 0 ]; do ;; --prefix|--root) if [ "$#" -lt 2 ]; then - echo "error: --prefix requires a directory" >&2 + error "--prefix requires a directory" exit 1 fi prefix=$2 @@ -101,7 +109,7 @@ while [ "$#" -gt 0 ]; do exit 0 ;; *) - echo "error: unknown option: $1" >&2 + error "unknown option: $1" usage >&2 exit 1 ;; @@ -110,7 +118,7 @@ done need_cmd() { if ! command -v "$1" >/dev/null 2>&1; then - echo "error: required command not found: $1" >&2 + error "required command not found: $1" exit 1 fi } @@ -170,10 +178,10 @@ resolve_source() { if [ -n "$src" ] && [ -e "$src" ]; then if ! is_checkout "$src" || ! git -C "$src" rev-parse --git-dir >/dev/null 2>&1; then - echo "error: $src exists but is not a dstack git checkout" >&2 + error "$src exists but is not a dstack git checkout" exit 1 fi - echo "updating dstack source in $src" + info "updating dstack source in $src" ( cd "$src" git fetch --tags origin @@ -181,26 +189,26 @@ resolve_source() { if git rev-parse --verify "origin/$ref" >/dev/null 2>&1; then git pull --ff-only origin "$ref" fi - ) + ) >&2 elif [ -n "$src" ]; then - echo "cloning dstack source into $src" - git clone "$repo" "$src" + info "cloning dstack source into $src" + git clone "$repo" "$src" >&2 ( cd "$src" git fetch --tags origin git checkout "$ref" - ) + ) >&2 else need_cmd mktemp tmp_src=$(mktemp -d "${TMPDIR:-/tmp}/dstack-install.XXXXXX") src="$tmp_src/source" - echo "cloning dstack source into a temporary checkout" - git clone "$repo" "$src" + info "cloning dstack source into a temporary checkout" + git clone "$repo" "$src" >&2 ( cd "$src" git fetch --tags origin git checkout "$ref" - ) + ) >&2 fi abs_dir "$src" @@ -210,17 +218,17 @@ validate_prefix() { case "$prefix" in /*) ;; *) - echo "error: --prefix must be an absolute path" >&2 + error "--prefix must be an absolute path" exit 1 ;; esac if [ "$prefix" = "/" ]; then - echo "error: --prefix must not be /" >&2 + error "--prefix must not be /" exit 1 fi case "$prefix" in *"/../"*|*"/.."|*"/./"*|*"/.") - echo "error: --prefix must not contain . or .. path components" >&2 + error "--prefix must not contain . or .. path components" exit 1 ;; esac @@ -255,7 +263,7 @@ else install -d -m 0755 "$bin_dir" fi -echo "building dstackup from $checkout" +info "building dstackup from $checkout" ( cd "$core_checkout" cargo build --release \ @@ -266,7 +274,7 @@ install_bin() { src_bin="$core_checkout/target/release/$1" dest_bin="$bin_dir/$2" if [ ! -f "$src_bin" ]; then - echo "error: expected binary not found: $src_bin" >&2 + error "expected binary not found: $src_bin" exit 1 fi if [ -n "$sudo_cmd" ]; then diff --git a/dstack/scripts/setup-bridge.sh b/dstack/scripts/setup-bridge.sh index f6fbf2247..ecc3d2901 100755 --- a/dstack/scripts/setup-bridge.sh +++ b/dstack/scripts/setup-bridge.sh @@ -44,83 +44,6 @@ run_cmd() { fi } -# --- Detect qemu-bridge-helper path --- - -find_bridge_helper() { - local paths=( - /usr/lib/qemu/qemu-bridge-helper - /usr/libexec/qemu-bridge-helper - /usr/local/lib/qemu/qemu-bridge-helper - /usr/local/libexec/qemu-bridge-helper - ) - for p in "${paths[@]}"; do - if [[ -f "$p" ]]; then - echo "$p" - return 0 - fi - done - return 1 -} - -# --- Detect current bridge provider --- - -# Returns "libvirt:<net_name>" if bridge is managed by a libvirt network, -# "standalone" otherwise. -detect_bridge_provider() { - if command -v virsh &>/dev/null; then - local name br - while read -r name; do - [[ -z "$name" ]] && continue - br=$(virsh net-dumpxml "$name" 2>/dev/null | grep -oP "<bridge name='\K[^']*" | head -1 || true) - if [[ "$br" == "$BRIDGE" ]]; then - echo "libvirt:$name" - return 0 - fi - done < <(virsh net-list --all --name 2>/dev/null) - fi - echo "standalone" -} - -# --- Check functions --- - -check_bridge_helper() { - echo - bold "qemu-bridge-helper" - local helper - if ! helper=$(find_bridge_helper); then - check_fail "qemu-bridge-helper not found" - check_info "Install QEMU: sudo apt install qemu-system-x86" - return - fi - check_pass "found at $helper" - - if [[ -u "$helper" ]]; then - check_pass "setuid bit is set" - else - check_fail "setuid bit not set" - check_info "Fix: sudo chmod u+s $helper" - fi -} - -check_bridge_conf() { - echo - bold "/etc/qemu/bridge.conf" - local conf="/etc/qemu/bridge.conf" - if [[ ! -f "$conf" ]]; then - check_fail "$conf does not exist" - check_info "Fix: sudo mkdir -p /etc/qemu && echo 'allow $BRIDGE' | sudo tee $conf" - return - fi - check_pass "$conf exists" - - if grep -qE "^allow[[:space:]]+($BRIDGE|all)[[:space:]]*$" "$conf" 2>/dev/null; then - check_pass "bridge '$BRIDGE' is allowed" - else - check_fail "bridge '$BRIDGE' not found in $conf" - check_info "Fix: echo 'allow $BRIDGE' | sudo tee -a $conf" - fi -} - check_bridge_interface() { echo bold "bridge interface: $BRIDGE" @@ -192,47 +115,6 @@ check_dhcp() { check_info "Run: $(basename "$0") setup --mode standalone --bridge $BRIDGE" } -check_dhcp_notify() { - echo - bold "DHCP lease notification" - - local provider - provider=$(detect_bridge_provider) - - if [[ "$provider" == libvirt:* ]]; then - check_warn "libvirt DHCP does not support dhcp-script callback" - check_info "port forwarding requires manual PRPC call or alternative notification" - return - fi - - # Check dnsmasq config for dhcp-script - local conf_files=(/etc/dnsmasq.d/*"$BRIDGE"* /etc/dnsmasq.d/*.conf) - local found_script="" - for f in "${conf_files[@]}"; do - [[ -f "$f" ]] || continue - local script_path - script_path=$(grep -oP '^dhcp-script=\K.*' "$f" 2>/dev/null || true) - if [[ -n "$script_path" ]]; then - found_script="$script_path" - break - fi - done - - if [[ -z "$found_script" ]]; then - check_warn "no dhcp-script configured in dnsmasq" - check_info "port forwarding will not be set up automatically" - check_info "add 'dhcp-script=/usr/local/bin/dhcp-notify.sh' to dnsmasq config" - return - fi - - if [[ -x "$found_script" ]]; then - check_pass "dhcp-script configured: $found_script" - else - check_fail "dhcp-script $found_script is not executable or missing" - check_info "Fix: sudo chmod +x $found_script" - fi -} - check_ip_forward() { echo bold "IP forwarding" @@ -367,33 +249,6 @@ check_forward_rules() { # --- Setup: common --- -setup_bridge_conf() { - echo - bold "Setting up /etc/qemu/bridge.conf" - run_cmd sudo mkdir -p /etc/qemu - if [[ -f /etc/qemu/bridge.conf ]] && grep -qE "^allow[[:space:]]+($BRIDGE|all)" /etc/qemu/bridge.conf 2>/dev/null; then - echo " already configured" - else - run_cmd bash -c "echo 'allow $BRIDGE' | sudo tee -a /etc/qemu/bridge.conf" - fi -} - -setup_bridge_helper() { - echo - bold "Setting up qemu-bridge-helper" - local helper - if ! helper=$(find_bridge_helper); then - echo " $(red 'ERROR'): qemu-bridge-helper not found. Install QEMU first." - return 1 - fi - if [[ -u "$helper" ]]; then - echo " setuid already set on $helper" - else - run_cmd sudo chmod u+s "$helper" - echo " setuid set on $helper" - fi -} - setup_ip_forward() { echo bold "Enabling IP forwarding" @@ -542,28 +397,12 @@ sudo mv /tmp/.dstack-br-network $network" dhcp_end="${prefix}.254" fi - # Install dhcp-notify.sh if present - local notify_script="/usr/local/bin/dhcp-notify.sh" - local dhcp_script_line="" - local src_notify - src_notify="$(cd "$(dirname "$0")" && pwd)/dhcp-notify.sh" - if [[ -f "$src_notify" ]]; then - run_cmd sudo cp "$src_notify" "$notify_script" - run_cmd sudo chmod +x "$notify_script" - dhcp_script_line="dhcp-script=${notify_script}" - echo " installed $notify_script" - else - echo " $(yellow '[WARN]') dhcp-notify.sh not found at $src_notify" - echo " VM port forwarding will not be set up automatically" - fi - run_cmd bash -c "cat > /tmp/.dstack-dnsmasq <<HEREDOC interface=$BRIDGE bind-interfaces dhcp-range=${dhcp_start},${dhcp_end},255.255.255.0,12h dhcp-option=option:router,${bridge_ip} dhcp-option=option:dns-server,8.8.8.8,1.1.1.1 -${dhcp_script_line} HEREDOC sudo mv /tmp/.dstack-dnsmasq $conf" echo " created $conf" @@ -738,11 +577,8 @@ cmd_check() { echo "provider: $(bold 'standalone')" fi - check_bridge_helper - check_bridge_conf check_bridge_interface check_dhcp - check_dhcp_notify check_dhcp_firewall check_ip_forward check_nat_rules @@ -786,8 +622,6 @@ cmd_setup() { $DRY_RUN && echo "dry-run: $(yellow 'yes')" # Common setup - setup_bridge_conf - setup_bridge_helper setup_ip_forward # Mode-specific setup @@ -870,7 +704,9 @@ cmd_destroy() { fi fi - # Remove bridge.conf entry + # Remove the bridge.conf entry an older setup added. netd owns every host + # interface now, so qemu-bridge-helper is not used and the `allow` line is + # a standing grant to attach any local user's TAP to the bridge. local conf="/etc/qemu/bridge.conf" if [[ -f "$conf" ]] && grep -qE "^allow[[:space:]]+${BRIDGE}[[:space:]]*$" "$conf" 2>/dev/null; then echo diff --git a/dstack/serde-duration/Cargo.toml b/dstack/serde-duration/Cargo.toml index c8b9280c4..45dddcf22 100644 --- a/dstack/serde-duration/Cargo.toml +++ b/dstack/serde-duration/Cargo.toml @@ -14,3 +14,6 @@ repository.workspace = true [dependencies] serde = { workspace = true, features = ["derive", "alloc"] } + +[dev-dependencies] +serde_json = { workspace = true, features = ["std"] } diff --git a/dstack/serde-duration/src/lib.rs b/dstack/serde-duration/src/lib.rs index 345cc4f6b..2ece41091 100644 --- a/dstack/serde-duration/src/lib.rs +++ b/dstack/serde-duration/src/lib.rs @@ -5,12 +5,10 @@ use serde::{Deserialize, Deserializer, Serializer}; use std::time::Duration; -pub fn serialize<S>(duration: &Duration, serializer: S) -> Result<S::Ok, S::Error> -where - S: Serializer, -{ +/// Render a duration in the largest unit that divides it evenly. +fn format(duration: &Duration) -> String { if duration == &Duration::MAX { - return serializer.serialize_str("never"); + return "never".to_string(); } let (value, unit) = if duration.as_secs().is_multiple_of(24 * 3600) { (duration.as_secs() / (24 * 3600), "d") @@ -21,7 +19,49 @@ where } else { (duration.as_secs(), "s") }; - serializer.serialize_str(&format!("{}{}", value, unit)) + format!("{value}{unit}") +} + +/// Parse a `<value><unit>` duration, or `never` for [`Duration::MAX`]. +fn parse(s: &str) -> Result<Duration, String> { + if s.is_empty() { + return Err("Duration string cannot be empty".to_string()); + } + if s == "never" { + return Ok(Duration::MAX); + } + // Split on the last *character*, not the last byte: `split_at(len - 1)` + // panics inside `core::str` when the unit is multi-byte, and a config file + // is exactly the place where a typo should be reported, not panicked on. + let unit_start = s + .char_indices() + .next_back() + .map(|(i, _)| i) + .ok_or_else(|| "Duration string cannot be empty".to_string())?; + let (value, unit) = s.split_at(unit_start); + let value = value.parse::<u64>().map_err(|e| e.to_string())?; + + let multiplier = match unit { + "s" => 1, + "m" => 60, + "h" => 3600, + "d" => 24 * 3600, + _ => return Err("Invalid time unit. Use s, m, h, or d".to_string()), + }; + // Checked: `18446744073709551615d` would otherwise wrap to a short duration + // in release builds, i.e. a timeout far tighter than the operator asked for. + let seconds = value + .checked_mul(multiplier) + .ok_or_else(|| format!("Duration {s} is too large"))?; + + Ok(Duration::from_secs(seconds)) +} + +pub fn serialize<S>(duration: &Duration, serializer: S) -> Result<S::Ok, S::Error> +where + S: Serializer, +{ + serializer.serialize_str(&format(duration)) } pub fn deserialize<'de, D>(deserializer: D) -> Result<Duration, D::Error> @@ -29,26 +69,116 @@ where D: Deserializer<'de>, { let s = String::deserialize(deserializer)?; - if s.is_empty() { - return Err(serde::de::Error::custom("Duration string cannot be empty")); + parse(&s).map_err(serde::de::Error::custom) +} + +/// `Option<Duration>` variant, for config fields where an absent value means +/// "this knob is not configured" rather than zero. +/// +/// Prefer this over a sentinel (`0`, or the `never` string) whenever the field +/// is one of several independent gates: with a sentinel the same value has to +/// mean both "disabled" and a legitimate setting, which is exactly the +/// ambiguity that makes multi-gate configs hard to reason about. +pub mod option { + use super::{format, parse}; + use serde::{Deserialize, Deserializer, Serializer}; + use std::time::Duration; + + pub fn serialize<S>(duration: &Option<Duration>, serializer: S) -> Result<S::Ok, S::Error> + where + S: Serializer, + { + match duration { + Some(duration) => serializer.serialize_str(&format(duration)), + None => serializer.serialize_none(), + } } - if s == "never" { - return Ok(Duration::MAX); + + pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<Duration>, D::Error> + where + D: Deserializer<'de>, + { + let Some(s) = Option::<String>::deserialize(deserializer)? else { + return Ok(None); + }; + parse(&s).map(Some).map_err(serde::de::Error::custom) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::{Deserialize, Serialize}; + + #[derive(Debug, Serialize, Deserialize, PartialEq)] + struct Plain { + #[serde(with = "crate")] + d: Duration, } - let (value, unit) = s.split_at(s.len() - 1); - let value = value.parse::<u64>().map_err(serde::de::Error::custom)?; - - let seconds = match unit { - "s" => value, - "m" => value * 60, - "h" => value * 3600, - "d" => value * 24 * 3600, - _ => { - return Err(serde::de::Error::custom( - "Invalid time unit. Use s, m, h, or d", - )) + + #[derive(Debug, Serialize, Deserialize, PartialEq)] + struct Opt { + #[serde(default, with = "crate::option")] + d: Option<Duration>, + } + + #[test] + fn parses_every_unit() { + for (text, secs) in [("30s", 30), ("5m", 300), ("2h", 7200), ("1d", 86400)] { + let v: Plain = serde_json::from_str(&format!(r#"{{"d":"{text}"}}"#)).unwrap(); + assert_eq!(v.d, Duration::from_secs(secs)); } - }; + let v: Plain = serde_json::from_str(r#"{"d":"never"}"#).unwrap(); + assert_eq!(v.d, Duration::MAX); + } - Ok(Duration::from_secs(seconds)) + #[test] + fn rejects_bad_input() { + for text in ["", "5x", "abcs"] { + let parsed: Result<Plain, _> = serde_json::from_str(&format!(r#"{{"d":"{text}"}}"#)); + assert!(parsed.is_err(), "{text} should not parse"); + } + } + + #[test] + fn option_absent_is_none() { + let v: Opt = serde_json::from_str("{}").unwrap(); + assert_eq!(v.d, None); + let v: Opt = serde_json::from_str(r#"{"d":null}"#).unwrap(); + assert_eq!(v.d, None); + } + + #[test] + fn option_present_parses() { + let v: Opt = serde_json::from_str(r#"{"d":"5s"}"#).unwrap(); + assert_eq!(v.d, Some(Duration::from_secs(5))); + } + + #[test] + fn round_trips() { + let v = Opt { + d: Some(Duration::from_secs(300)), + }; + let text = serde_json::to_string(&v).unwrap(); + assert_eq!(text, r#"{"d":"5m"}"#); + assert_eq!(serde_json::from_str::<Opt>(&text).unwrap(), v); + } + + /// A bad unit is a config typo, so it has to come back as an error -- + /// including when the typo is not ASCII, where splitting on the last byte + /// used to panic inside `core::str`. + #[test] + fn a_bad_unit_is_an_error_not_a_panic() { + assert!(parse("5\u{5206}").is_err()); + assert!(parse("5x").is_err()); + assert!(parse("\u{5206}").is_err()); + assert!(parse("s").is_err()); + } + + /// Overflow would silently wrap to a much shorter timeout than asked for. + #[test] + fn an_oversized_duration_is_rejected() { + assert!(parse(&format!("{}d", u64::MAX)).is_err()); + assert_eq!(parse("5m").unwrap(), Duration::from_secs(300)); + } } diff --git a/dstack/sev-snp-attest/src/lib.rs b/dstack/sev-snp-attest/src/lib.rs index 04a2f9cca..1226070b4 100644 --- a/dstack/sev-snp-attest/src/lib.rs +++ b/dstack/sev-snp-attest/src/lib.rs @@ -127,12 +127,17 @@ fn get_report_configfs(report_data: [u8; 64]) -> Result<SnpQuote> { } fn write_first_existing(paths: &[std::path::PathBuf], binary: &[u8], hex: &[u8]) -> Result<()> { + use std::io::Write; let mut last_err = None; for path in paths { if !path.exists() { continue; } - match fs_err::write(path, binary).or_else(|_| fs_err::write(path, hex)) { + let write = |data: &[u8]| -> std::io::Result<()> { + let mut file = std::fs::OpenOptions::new().write(true).open(path)?; + file.write_all(data) + }; + match write(binary).or_else(|_| write(hex)) { Ok(()) => return Ok(()), Err(err) => last_err = Some(err), } diff --git a/dstack/sev-snp-qvl/Cargo.toml b/dstack/sev-snp-qvl/Cargo.toml index 58e4166d7..4d16e4500 100644 --- a/dstack/sev-snp-qvl/Cargo.toml +++ b/dstack/sev-snp-qvl/Cargo.toml @@ -17,3 +17,7 @@ moka.workspace = true reqwest.workspace = true sev.workspace = true tokio = { workspace = true, features = ["rt", "time"] } +x509-parser = { workspace = true, features = ["verify"] } +rustls-pki-types.workspace = true +rustls-webpki = { workspace = true, features = ["ring"] } +pem.workspace = true diff --git a/dstack/sev-snp-qvl/src/lib.rs b/dstack/sev-snp-qvl/src/lib.rs index 75add0834..1fa4da6f5 100644 --- a/dstack/sev-snp-qvl/src/lib.rs +++ b/dstack/sev-snp-qvl/src/lib.rs @@ -10,7 +10,7 @@ //! must still bind the verified measurement to app/config identity before //! production key release. -use std::{thread, time::Duration}; +use std::time::Duration; use anyhow::{anyhow, bail, Context, Result}; use moka::sync::Cache; @@ -27,8 +27,7 @@ const VLEK_CERT_GUID: [u8; 16] = [ 0xa8, 0x07, 0x4b, 0xc2, 0xa2, 0x5a, 0x48, 0x3e, 0xaa, 0xe6, 0x39, 0xc0, 0x45, 0xa0, 0xb8, 0xa1, ]; const CERT_TABLE_ENTRY_SIZE: usize = 24; -const AMD_KDS_BASE_URL_ENV: &str = "DSTACK_AMD_KDS_BASE_URL"; -const AMD_KDS_DEFAULT_BASE_URL: &str = "https://kdsintf.amd.com/vcek/v1"; +pub const AMD_KDS_DEFAULT_BASE_URL: &str = "https://kdsintf.amd.com/vcek/v1"; const AMD_KDS_CONNECT_TIMEOUT: Duration = Duration::from_secs(30); const AMD_KDS_REQUEST_TIMEOUT: Duration = Duration::from_secs(120); const AMD_KDS_CA_CACHE_CAPACITY: u64 = 16; @@ -41,6 +40,136 @@ pub enum AmdSnpProduct { Turin, } +/// AMD SEV-SNP quote verifier with caller-owned ARK trust anchors. +/// +/// One ARK is required per supported processor family because AMD operates a +/// separate hierarchy for each family. `new_prod` uses the ARKs shipped by the +/// `sev` crate; `new` accepts PEM encoded ARKs from the caller. +#[derive(Debug, Clone)] +pub struct QuoteVerifier { + milan_ark_pem: Vec<u8>, + genoa_ark_pem: Vec<u8>, + turin_ark_pem: Vec<u8>, + external_roots: [bool; 3], +} + +impl QuoteVerifier { + pub fn new(milan_ark_pem: Vec<u8>, genoa_ark_pem: Vec<u8>, turin_ark_pem: Vec<u8>) -> Self { + Self { + milan_ark_pem, + genoa_ark_pem, + turin_ark_pem, + external_roots: [true; 3], + } + } + + pub fn new_prod() -> Self { + Self { + milan_ark_pem: builtin::milan::ARK.to_vec(), + genoa_ark_pem: builtin::genoa::ARK.to_vec(), + turin_ark_pem: builtin::turin::ARK.to_vec(), + external_roots: [false; 3], + } + } + + /// Create a production verifier overriding one product-family ARK. + pub fn new_with_root(product: AmdSnpProduct, ark_pem: Vec<u8>) -> Self { + Self::new_prod().with_root(product, ark_pem) + } + + /// Override one product-family ARK. + pub fn with_root(mut self, product: AmdSnpProduct, ark_pem: Vec<u8>) -> Self { + match product { + AmdSnpProduct::Milan => { + self.milan_ark_pem = ark_pem; + self.external_roots[0] = true; + } + AmdSnpProduct::Genoa => { + self.genoa_ark_pem = ark_pem; + self.external_roots[1] = true; + } + AmdSnpProduct::Turin => { + self.turin_ark_pem = ark_pem; + self.external_roots[2] = true; + } + } + self + } + + fn ark(&self, product: AmdSnpProduct) -> CertBytes { + CertBytes { + bytes: match product { + AmdSnpProduct::Milan => self.milan_ark_pem.clone(), + AmdSnpProduct::Genoa => self.genoa_ark_pem.clone(), + AmdSnpProduct::Turin => self.turin_ark_pem.clone(), + }, + encoding: CertEncoding::Pem, + } + } + + fn uses_external_root(&self, product: AmdSnpProduct) -> bool { + self.external_roots[match product { + AmdSnpProduct::Milan => 0, + AmdSnpProduct::Genoa => 1, + AmdSnpProduct::Turin => 2, + }] + } + + pub fn verify( + &self, + report: &[u8], + cert_chain: &[Vec<u8>], + expected_report_data: &[u8; 64], + ) -> Result<VerifiedAmdSnpReport> { + let (ask, vcek) = normalize_ask_vcek_certs(cert_chain)?; + let verified = + verify_amd_snp_attestation_with_certs_and_arks(report, ask, vcek, |product| { + (self.ark(product), self.uses_external_root(product)) + })?; + if &verified.report_data != expected_report_data { + bail!("amd sev-snp report_data mismatch"); + } + Ok(verified) + } + + pub async fn fetch_and_verify( + &self, + kds_client: &AmdKdsClient, + report: &[u8], + cert_chain: &[Vec<u8>], + expected_report_data: &[u8; 64], + ) -> Result<VerifiedAmdSnpReport> { + if !cert_chain.is_empty() { + return self.verify(report, cert_chain, expected_report_data); + } + let report_obj = AttestationReport::from_bytes(report) + .map_err(|err| anyhow!("failed to parse amd sev-snp report: {err}"))?; + let mut errors = Vec::new(); + for product in amd_snp_product_candidates_for_report(&report_obj)? { + match kds_client + .fetch_collateral_for_product(product, &report_obj) + .await + { + Ok(collateral) => match verify_amd_snp_attestation_with_cert_chain( + report, + self.ark(product), + collateral.ask, + collateral.vcek, + self.uses_external_root(product), + ) { + Ok(verified) if &verified.report_data == expected_report_data => { + return Ok(verified); + } + Ok(_) => bail!("amd sev-snp report_data mismatch"), + Err(err) => errors.push(format!("{}: {err:#}", product.kds_name())), + }, + Err(err) => errors.push(format!("{}: {err:#}", product.kds_name())), + } + } + bail!("amd sev-snp verification failed: {}", errors.join("; ")) + } +} + impl AmdSnpProduct { fn kds_name(self) -> &'static str { match self { @@ -181,7 +310,7 @@ pub struct AmdKdsClient { impl AmdKdsClient { pub fn new() -> Result<Self> { - Self::with_base_url(amd_kds_base_url()) + Self::with_base_url(AMD_KDS_DEFAULT_BASE_URL) } pub fn with_base_url(base_url: impl AsRef<str>) -> Result<Self> { @@ -199,56 +328,6 @@ impl AmdKdsClient { }) } - pub async fn verify_evidence_with_kds_fallback( - &self, - report: &[u8], - cert_chain: &[Vec<u8>], - expected_report_data: &[u8; 64], - ) -> Result<VerifiedAmdSnpReport> { - if !cert_chain.is_empty() { - return verify_amd_snp_evidence(report, cert_chain, expected_report_data); - } - if report.len() != 1184 { - bail!( - "invalid amd sev-snp report length: expected 1184 bytes, got {}", - report.len() - ); - } - let report_obj = AttestationReport::from_bytes(report) - .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; - let collateral = self - .fetch_collateral_for_report(&report_obj) - .await - .context("failed to fetch amd sev-snp KDS collateral for empty cert_chain")?; - let verified = verify_amd_snp_attestation_with_cert_chain( - report, - collateral.ark, - collateral.ask, - collateral.vcek, - )?; - if &verified.report_data != expected_report_data { - bail!("amd sev-snp report_data mismatch"); - } - Ok(verified) - } - - async fn fetch_collateral_for_report( - &self, - report: &AttestationReport, - ) -> Result<AmdKdsCollateral> { - let mut errors = Vec::new(); - for product in amd_snp_product_candidates_for_report(report)? { - match self.fetch_collateral_for_product(product, report).await { - Ok(collateral) => return Ok(collateral), - Err(err) => errors.push(format!("{}: {err:#}", product.kds_name())), - } - } - bail!( - "amd sev-snp KDS collateral unavailable for supported products: {}", - errors.join("; ") - ) - } - async fn fetch_collateral_for_product( &self, product: AmdSnpProduct, @@ -400,6 +479,17 @@ fn verify_amd_snp_attestation_with_certs( report_bytes: &[u8], ask_bytes: CertBytes, vcek_bytes: CertBytes, +) -> Result<VerifiedAmdSnpReport> { + verify_amd_snp_attestation_with_certs_and_arks(report_bytes, ask_bytes, vcek_bytes, |product| { + (product.builtin_ark(), false) + }) +} + +fn verify_amd_snp_attestation_with_certs_and_arks( + report_bytes: &[u8], + ask_bytes: CertBytes, + vcek_bytes: CertBytes, + ark_for_product: impl Fn(AmdSnpProduct) -> (CertBytes, bool), ) -> Result<VerifiedAmdSnpReport> { if report_bytes.len() != 1184 { bail!( @@ -411,11 +501,13 @@ fn verify_amd_snp_attestation_with_certs( .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; let mut errors = Vec::new(); for product in amd_snp_product_candidates_for_report(&report)? { + let (ark, external_root) = ark_for_product(product); match verify_amd_snp_attestation_with_cert_chain( report_bytes, - product.builtin_ark(), + ark, ask_bytes.clone(), vcek_bytes.clone(), + external_root, ) { Ok(verified) => return Ok(verified), Err(err) => errors.push(format!("{}: {err:#}", product.kds_name())), @@ -432,6 +524,7 @@ fn verify_amd_snp_attestation_with_cert_chain( ark_bytes: CertBytes, ask_bytes: CertBytes, vcek_bytes: CertBytes, + external_root: bool, ) -> Result<VerifiedAmdSnpReport> { if report_bytes.len() != 1184 { bail!( @@ -445,14 +538,16 @@ fn verify_amd_snp_attestation_with_cert_chain( let ark = parse_certificate(&ark_bytes, "ark")?; let ask = parse_certificate(&ask_bytes, "ask")?; let vcek = parse_certificate(&vcek_bytes, "vcek")?; - - let chain = Chain { - ca: ca::Chain { ark, ask }, - vek: vcek.clone(), - }; - chain + if external_root { + verify_x509_chain(&ark_bytes, &ask_bytes, &vcek_bytes)?; + } else { + Chain { + ca: ca::Chain { ark, ask }, + vek: vcek.clone(), + } .verify() .map_err(|err| anyhow::anyhow!("amd cert chain verification failed: {err:?}"))?; + } (&vcek, &report).verify().map_err(|err| { anyhow::anyhow!("amd sev-snp report signature verification failed: {err:?}") })?; @@ -474,6 +569,97 @@ fn verify_amd_snp_attestation_with_cert_chain( }) } +fn verify_x509_chain(ark: &CertBytes, ask: &CertBytes, vcek: &CertBytes) -> Result<()> { + use x509_parser::prelude::{FromDer, X509Certificate}; + + let ark_der = certificate_der(ark)?; + let ask_der = certificate_der(ask)?; + let vcek_der = certificate_der(vcek)?; + let (_, ark) = X509Certificate::from_der(&ark_der).context("failed to parse ARK X.509")?; + let (_, ask) = X509Certificate::from_der(&ask_der).context("failed to parse ASK X.509")?; + let (_, vcek) = X509Certificate::from_der(&vcek_der).context("failed to parse VCEK X.509")?; + if !ark.is_ca() || !ask.is_ca() { + bail!("AMD ARK and ASK certificates must be certificate authorities"); + } + if ark.subject() != ark.issuer() + || ask.issuer() != ark.subject() + || vcek.issuer() != ask.subject() + { + bail!("AMD certificate issuer/subject chain mismatch"); + } + verify_certificate_signature(&ark_der, &ark) + .context("ARK self-signature verification failed")?; + verify_certificate_signature(&ark_der, &ask).context("ASK signature verification failed")?; + verify_certificate_signature(&ask_der, &vcek).context("VCEK signature verification failed")?; + Ok(()) +} + +fn verify_certificate_signature( + issuer_der: &[u8], + certificate: &x509_parser::certificate::X509Certificate<'_>, +) -> Result<()> { + use rustls_pki_types::CertificateDer; + const RSA_PSS_OID: &str = "1.2.840.113549.1.1.10"; + const ECDSA_SHA384_OID: &str = "1.2.840.10045.4.3.3"; + const SHA384_OID: &str = "2.16.840.1.101.3.4.2.2"; + const MGF1_OID: &str = "1.2.840.113549.1.1.8"; + + anyhow::ensure!( + certificate.signature_algorithm == certificate.tbs_certificate.signature, + "certificate signatureAlgorithm does not match TBSCertificate signature" + ); + let oid = certificate.signature_algorithm.algorithm.to_id_string(); + let algorithm = match oid.as_str() { + RSA_PSS_OID => { + use x509_parser::signature_algorithm::SignatureAlgorithm; + let SignatureAlgorithm::RSASSA_PSS(params) = + SignatureAlgorithm::try_from(&certificate.signature_algorithm) + .context("invalid RSA-PSS signature parameters")? + else { + bail!("invalid RSA-PSS signature parameters"); + }; + let mask = params + .mask_gen_algorithm() + .context("invalid RSA-PSS mask generation parameters")?; + anyhow::ensure!( + params.hash_algorithm_oid().to_id_string() == SHA384_OID + && mask.mgf.to_id_string() == MGF1_OID + && mask.hash.to_id_string() == SHA384_OID + && params.salt_length() == 48 + && params.trailer_field() == 1, + "RSA-PSS certificate signature must use SHA-384, MGF1-SHA384, and a 48-byte salt" + ); + webpki::ring::RSA_PSS_2048_8192_SHA384_LEGACY_KEY + } + ECDSA_SHA384_OID => { + anyhow::ensure!( + certificate.signature_algorithm.parameters.is_none(), + "ECDSA-SHA384 signature parameters must be absent" + ); + webpki::ring::ECDSA_P384_SHA384 + } + _ => bail!("unsupported certificate signature algorithm: {oid}"), + }; + let issuer_der = CertificateDer::from(issuer_der); + let issuer = webpki::EndEntityCert::try_from(&issuer_der) + .context("failed to parse certificate issuer")?; + let message = certificate.tbs_certificate.as_ref(); + let signature = certificate.signature_value.data.as_ref(); + issuer + .verify_signature(algorithm, message, signature) + .context("invalid certificate signature")?; + Ok(()) +} + +fn certificate_der(cert: &CertBytes) -> Result<Vec<u8>> { + match cert.encoding { + CertEncoding::Der => Ok(cert.bytes.clone()), + CertEncoding::Pem => Ok(::pem::parse(&cert.bytes) + .context("failed to parse certificate PEM")? + .into_contents()), + } +} + pub fn verify_amd_snp_evidence( report: &[u8], cert_chain: &[Vec<u8>], @@ -487,51 +673,6 @@ pub fn verify_amd_snp_evidence( Ok(verified) } -pub fn verify_amd_snp_evidence_with_kds_fallback( - report: &[u8], - cert_chain: &[Vec<u8>], - expected_report_data: &[u8; 64], -) -> Result<VerifiedAmdSnpReport> { - if !cert_chain.is_empty() { - return verify_amd_snp_evidence(report, cert_chain, expected_report_data); - } - - let report = report.to_vec(); - let expected_report_data = *expected_report_data; - thread::spawn(move || -> Result<VerifiedAmdSnpReport> { - let kds_client = AmdKdsClient::new()?; - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .context("failed to create amd sev-snp KDS fallback runtime")?; - runtime.block_on(kds_client.verify_evidence_with_kds_fallback( - &report, - &[], - &expected_report_data, - )) - }) - .join() - .map_err(|_| anyhow!("amd sev-snp KDS fallback worker panicked"))? -} - -pub async fn verify_amd_snp_evidence_with_kds_fallback_async( - report: &[u8], - cert_chain: &[Vec<u8>], - expected_report_data: &[u8; 64], -) -> Result<VerifiedAmdSnpReport> { - AmdKdsClient::new()? - .verify_evidence_with_kds_fallback(report, cert_chain, expected_report_data) - .await -} - -fn amd_kds_base_url() -> String { - std::env::var(AMD_KDS_BASE_URL_ENV) - .ok() - .map(|url| url.trim().trim_end_matches('/').to_string()) - .filter(|url| !url.is_empty()) - .unwrap_or_else(|| AMD_KDS_DEFAULT_BASE_URL.to_string()) -} - fn normalize_amd_kds_base_url(base_url: &str) -> Result<String> { let base_url = base_url.trim().trim_end_matches('/').to_string(); if base_url.is_empty() { diff --git a/dstack/size-parser/src/lib.rs b/dstack/size-parser/src/lib.rs index b7cf39896..b9ed4b2d1 100644 --- a/dstack/size-parser/src/lib.rs +++ b/dstack/size-parser/src/lib.rs @@ -46,8 +46,8 @@ pub enum MemorySizeError { InvalidHex(String), #[error("Invalid numeric value: {0}")] InvalidNumber(String), - #[error("Unknown memory size suffix: {0}")] - UnknownSuffix(char), + #[error("Unknown memory size unit: {0}")] + UnknownUnit(String), #[error("Overflow in memory size calculation")] Overflow, } @@ -95,13 +95,19 @@ impl MemorySize { /// Supports the following formats: /// - Plain numbers: "1024", "2048" /// - Hexadecimal: "0x1000", "0X2000" - /// - With suffixes: "2K", "4M", "1G", "2T" (case-insensitive) + /// - With suffixes: "2K", "4M", "1G", "2T", "1P" (case-insensitive) + /// - Two- and three-letter spellings of the same units: "4MB", "1GiB", + /// "2TB", "1PiB", and "B" as a bytes suffix (e.g. "512B") /// - /// Suffixes use binary (1024-based) multipliers: - /// - K/k: 1024 bytes - /// - M/m: 1024^2 bytes - /// - G/g: 1024^3 bytes - /// - T/t: 1024^4 bytes + /// Every suffix is a binary (1024-based) multiplier, including the "B" and + /// "iB" spellings. "1KB" is 1024 bytes, not 1000: the crate names sizes the + /// way QEMU and the kernel do, and silently switching to powers of ten for + /// one spelling would be worse than being consistently binary. + /// - K/KB/KiB: 1024 bytes + /// - M/MB/MiB: 1024^2 bytes + /// - G/GB/GiB: 1024^3 bytes + /// - T/TB/TiB: 1024^4 bytes + /// - P/PB/PiB: 1024^5 bytes pub fn parse(s: &str) -> Result<Self, MemorySizeError> { let s = s.trim(); @@ -125,22 +131,30 @@ impl MemorySize { return Ok(Self::from_bytes(bytes)); } - // Handle numbers with suffixes - let Some(last_char) = s.chars().last() else { - return Err(MemorySizeError::Empty); - }; + // Handle numbers with suffixes. Split at the first non-digit rather + // than trimming the trailing character: trim_end_matches removes every + // repetition, so "1GG" used to lose both and parse as 1 GiB. + let split = s + .find(|c: char| !c.is_ascii_digit()) + .ok_or_else(|| MemorySizeError::InvalidNumber(s.to_string()))?; + let (num_part, unit) = s.split_at(split); + // No digits at all is a malformed number, not a malformed unit. The + // previous implementation classified "abc" by its last character and + // called it an unknown suffix, which pointed at the wrong half. + if num_part.is_empty() { + return Err(MemorySizeError::InvalidNumber(s.to_string())); + } - let multiplier = match last_char.to_ascii_lowercase() { - 'k' => 1024u64, - 'm' => 1024u64.saturating_mul(1024), - 'g' => 1024u64.saturating_mul(1024).saturating_mul(1024), - 't' => 1024u64 - .saturating_mul(1024) - .saturating_mul(1024) - .saturating_mul(1024), - _ => return Err(MemorySizeError::UnknownSuffix(last_char)), + const KIB: u64 = 1024; + let multiplier = match unit.to_ascii_lowercase().as_str() { + "b" => 1, + "k" | "kb" | "kib" => KIB, + "m" | "mb" | "mib" => KIB.pow(2), + "g" | "gb" | "gib" => KIB.pow(3), + "t" | "tb" | "tib" => KIB.pow(4), + "p" | "pb" | "pib" => KIB.pow(5), + _ => return Err(MemorySizeError::UnknownUnit(unit.to_string())), }; - let num_part = s.trim_end_matches(last_char); let num = num_part .parse::<u64>() .map_err(|_| MemorySizeError::InvalidNumber(num_part.to_string()))?; @@ -155,6 +169,7 @@ impl MemorySize { /// Format the memory size in a human-readable way pub fn format_human(&self) -> String { const UNITS: &[(&str, u64)] = &[ + ("P", 1024u64.pow(5)), ("T", 1024u64.pow(4)), ("G", 1024u64.pow(3)), ("M", 1024u64.pow(2)), @@ -438,6 +453,43 @@ mod tests { assert_eq!(MemorySize::parse("1M").unwrap().bytes(), 1024 * 1024); assert_eq!(MemorySize::parse("2m").unwrap().bytes(), 2 * 1024 * 1024); assert_eq!(MemorySize::parse("1G").unwrap().bytes(), 1024 * 1024 * 1024); + // Petabytes, and the two- and three-letter spellings of every unit. + assert_eq!(MemorySize::parse("1P").unwrap().bytes(), 1024u64.pow(5)); + assert_eq!(MemorySize::parse("1p").unwrap().bytes(), 1024u64.pow(5)); + assert_eq!(MemorySize::parse("1PB").unwrap().bytes(), 1024u64.pow(5)); + assert_eq!(MemorySize::parse("1PiB").unwrap().bytes(), 1024u64.pow(5)); + assert_eq!(MemorySize::parse("1pib").unwrap().bytes(), 1024u64.pow(5)); + assert_eq!( + MemorySize::parse("2TB").unwrap().bytes(), + 2 * 1024u64.pow(4) + ); + assert_eq!( + MemorySize::parse("2TiB").unwrap().bytes(), + 2 * 1024u64.pow(4) + ); + assert_eq!( + MemorySize::parse("4GB").unwrap().bytes(), + 4 * 1024u64.pow(3) + ); + assert_eq!( + MemorySize::parse("4MiB").unwrap().bytes(), + 4 * 1024u64.pow(2) + ); + assert_eq!(MemorySize::parse("8KB").unwrap().bytes(), 8 * 1024); + assert_eq!(MemorySize::parse("512B").unwrap().bytes(), 512); + // "KB" is binary here, matching every other spelling in this crate. + assert_eq!(MemorySize::parse("1KB").unwrap().bytes(), 1024); + // The production pci_hole64_size value, in every accepted spelling. + let one_pib = 1125899906842624u64; + for spelling in ["1P", "1PiB", "1024T", "0x4000000000000", "1125899906842624"] { + assert_eq!( + MemorySize::parse(spelling).unwrap().bytes(), + one_pib, + "{spelling}" + ); + } + // A repeated suffix is rejected instead of silently losing a letter. + assert!(MemorySize::parse("1GG").is_err()); assert_eq!( MemorySize::parse("2g").unwrap().bytes(), 2 * 1024 * 1024 * 1024 @@ -455,7 +507,7 @@ mod tests { )); assert!(matches!( MemorySize::parse("abc"), - Err(MemorySizeError::UnknownSuffix('c')) + Err(MemorySizeError::InvalidNumber(_)) )); assert!(matches!( MemorySize::parse("0xgg"), diff --git a/dstack/supervisor/Cargo.toml b/dstack/supervisor/Cargo.toml index e9e91b559..3d7b3439d 100644 --- a/dstack/supervisor/Cargo.toml +++ b/dstack/supervisor/Cargo.toml @@ -16,7 +16,7 @@ bon.workspace = true clap = { workspace = true, features = ["derive", "string"] } dashmap.workspace = true fs-err.workspace = true -git-version.workspace = true +dstack-build-info.workspace = true libc.workspace = true load_config.workspace = true nix = { workspace = true, features = ["resource"] } diff --git a/dstack/supervisor/client/src/main.rs b/dstack/supervisor/client/src/main.rs index 4f50793e5..16d8c61a5 100644 --- a/dstack/supervisor/client/src/main.rs +++ b/dstack/supervisor/client/src/main.rs @@ -50,7 +50,11 @@ async fn main() -> Result<()> { { use tracing_subscriber::{fmt, EnvFilter}; let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); - fmt().with_env_filter(filter).with_ansi(false).init(); + fmt() + .with_env_filter(filter) + .with_ansi(false) + .with_writer(std::io::stderr) + .init(); } let cli = Cli::parse(); diff --git a/dstack/supervisor/src/main.rs b/dstack/supervisor/src/main.rs index 291ef3203..a84149c8c 100644 --- a/dstack/supervisor/src/main.rs +++ b/dstack/supervisor/src/main.rs @@ -23,13 +23,7 @@ pub fn load_config_figment(config_file: Option<&str>) -> Figment { } fn app_version() -> String { - const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION"); - const VERSION: &str = git_version::git_version!( - args = ["--abbrev=20", "--always", "--dirty=-modified"], - prefix = "git:", - fallback = "unknown" - ); - format!("v{CARGO_PKG_VERSION} ({VERSION})") + dstack_build_info::app_version!() } #[derive(Parser)] diff --git a/dstack/supervisor/src/process.rs b/dstack/supervisor/src/process.rs index c424f9b51..94e0c61e5 100644 --- a/dstack/supervisor/src/process.rs +++ b/dstack/supervisor/src/process.rs @@ -83,6 +83,17 @@ impl ProcessStateRT { pub(crate) fn is_started(&self) -> bool { self.started } + + /// An explicit stop may arrive just after the process exited on its own + /// (VM launchers exit cleanly after reaping their children). Report such + /// a clean exit as the intended stop, but keep non-zero exit codes and + /// errors visible for diagnostics. `stopped_at` recorded by the wait task + /// is left untouched. + fn normalize_clean_exit(&mut self) { + if matches!(self.status, ProcessStatus::Exited(0)) { + self.status = ProcessStatus::Stopped; + } + } } impl ProcessStateRT { @@ -289,8 +300,13 @@ impl Process { if is_running { bail!("Missing kill tx for process"); } + state.normalize_clean_exit(); return Ok(()); }; + if !is_running { + state.normalize_clean_exit(); + return Ok(()); + } match stop_tx.send(()) { Ok(()) => Ok(()), Err(()) => match is_running { diff --git a/dstack/supervisor/src/web_api.rs b/dstack/supervisor/src/web_api.rs index 2521ee20b..a3529b0d5 100644 --- a/dstack/supervisor/src/web_api.rs +++ b/dstack/supervisor/src/web_api.rs @@ -6,7 +6,7 @@ use anyhow::{anyhow, Result}; use or_panic::ResultOrPanic; use rocket::figment::Figment; use rocket::serde::json::Json; -use rocket::{delete, get, post, routes, Build, Rocket, State}; +use rocket::{delete, get, post, routes, Build, Rocket, Shutdown, State}; use serde::{Deserialize, Serialize}; use tokio::signal; use tracing::info; @@ -81,8 +81,12 @@ fn clear(supervisor: &State<Supervisor>) -> Json<Response<()>> { } #[post("/shutdown")] -async fn shutdown(supervisor: &State<Supervisor>) -> Json<Response<()>> { - to_json(perform_shutdown(supervisor, false).await) +async fn shutdown(supervisor: &State<Supervisor>, rocket_shutdown: Shutdown) -> Json<Response<()>> { + let result = supervisor.shutdown().await; + if result.is_ok() { + rocket_shutdown.notify(); + } + to_json(result) } async fn perform_shutdown(supervisor: &Supervisor, force: bool) -> Result<()> { diff --git a/dstack/tee-simulator/Cargo.toml b/dstack/tee-simulator/Cargo.toml index 1a7ce39db..b97d859a5 100644 --- a/dstack/tee-simulator/Cargo.toml +++ b/dstack/tee-simulator/Cargo.toml @@ -17,12 +17,33 @@ path = "src/main.rs" anyhow.workspace = true cc-eventlog.workspace = true clap.workspace = true +dstack-attest.workspace = true +dstack-types.workspace = true +dstack-mr.workspace = true fuser.workspace = true libc.workspace = true +nix = { workspace = true, features = ["fs", "mount", "user"] } sd-notify.workspace = true sha2.workspace = true +tpm2.workspace = true tracing.workspace = true tracing-subscriber.workspace = true +serde_json.workspace = true +fs-err.workspace = true +safe-write.workspace = true +hex.workspace = true +mock-attestation = { path = "../crates/mock-attestation" } +tokio = { workspace = true, features = ["full"] } +libloading = "0.8" +pem.workspace = true +aws-nitro-enclaves-nsm-api = "0.4" +serde_cbor = "0.11" [dev-dependencies] dcap-qvl.workspace = true +tempfile.workspace = true +reqwest.workspace = true +sev-snp-qvl.workspace = true +tpm-qvl.workspace = true +tpm-types.workspace = true +nsm-qvl.workspace = true diff --git a/dstack/tee-simulator/src/main.rs b/dstack/tee-simulator/src/main.rs index 0021ca16b..940a8eba3 100644 --- a/dstack/tee-simulator/src/main.rs +++ b/dstack/tee-simulator/src/main.rs @@ -2,31 +2,53 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::path::{Path, PathBuf}; +use std::{ + ffi::CString, + os::unix::{ffi::OsStrExt, fs::PermissionsExt as _}, + path::{Path, PathBuf}, +}; use anyhow::{bail, Context, Result}; -use clap::{Parser, ValueEnum}; +use clap::Parser; +use dstack_attest::{attestation::RootCaPaths, trust_anchors}; +use dstack_types::{TeeSimulatorConfig, TeeVariant}; use fuser::{Filesystem, MountOption, Session}; use tracing::info; +mod nsm; +mod sev_snp; mod tdx; +mod tpm; -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, ValueEnum)] -enum TeePlatform { - #[default] - Tdx, -} +/// Runtime-directory-relative location of [`trust_anchors::ANCHOR_DIR`]. +const TRUST_ANCHOR_SUBDIR: &str = "attestation"; + +/// Matches `MockCollateralState::new`, used when the host names no service. +const DEFAULT_COLLATERAL_BASE_URL: &str = "http://127.0.0.1:8088"; #[derive(Parser)] #[command(about = "Development-only simulator for Linux TEE guest ABIs")] struct Args { /// TEE platform ABI to simulate. - #[arg(long, value_enum, default_value = "tdx")] - platform: TeePlatform, + #[arg(long)] + platform: Option<TeeVariant>, + + /// Development-only simulator config. The systemd unit overrides this + /// standalone default with its early-mounted runtime path. + #[arg(long, default_value = "/dstack/.host-shared/.tee-simulator.json")] + config: PathBuf, /// Override the platform backend's default mountpoint. #[arg(long)] mountpoint: Option<PathBuf>, + + /// Runtime state directory (overridable by unprivileged E2E tests). + #[arg(long, default_value = "/run/dstack")] + runtime_dir: PathBuf, + + /// DMI sysfs directory. Tests may override this with a temporary directory. + #[arg(long, default_value = "/sys/class/dmi/id")] + dmi_root: PathBuf, } /// Platform-specific simulator backend. @@ -39,12 +61,15 @@ trait TeeBackend { const PLATFORM: &'static str; const DEFAULT_MOUNTPOINT: &'static str; - fn create_filesystem() -> Result<Self::Fs>; + fn create_filesystem(config: &TeeSimulatorConfig) -> Result<Self::Fs>; fn prepare_mountpoint(mountpoint: &Path) -> Result<()>; fn real_tee_available() -> bool; } -fn run_backend<B: TeeBackend>(mountpoint_override: Option<PathBuf>) -> Result<()> { +fn run_backend<B: TeeBackend>( + mountpoint_override: Option<PathBuf>, + config: &TeeSimulatorConfig, +) -> Result<()> { let default_mountpoint = Path::new(B::DEFAULT_MOUNTPOINT); let mountpoint = mountpoint_override.as_deref().unwrap_or(default_mountpoint); @@ -54,9 +79,16 @@ fn run_backend<B: TeeBackend>(mountpoint_override: Option<PathBuf>) -> Result<() B::PLATFORM ); } + if is_mounted(mountpoint)? { + bail!( + "refusing to mount the {} simulator over an existing mount at {}", + B::PLATFORM, + mountpoint.display() + ); + } B::prepare_mountpoint(mountpoint)?; - let fs = B::create_filesystem()?; + let fs = B::create_filesystem(config)?; let mut session = Session::new( fs, mountpoint, @@ -81,15 +113,187 @@ fn run_backend<B: TeeBackend>(mountpoint_override: Option<PathBuf>) -> Result<() Ok(()) } +fn is_mounted(path: &Path) -> Result<bool> { + if !path.exists() { + return Ok(false); + } + let target = path + .canonicalize() + .with_context(|| format!("failed to resolve mountpoint {}", path.display()))?; + let mountinfo = fs_err::read_to_string("/proc/self/mountinfo") + .context("failed to read process mount table")?; + Ok(mountinfo.lines().any(|line| { + line.split_whitespace() + .nth(4) + .map(decode_mountinfo_path) + .is_some_and(|mounted| Path::new(&mounted) == target) + })) +} + +fn decode_mountinfo_path(value: &str) -> String { + value + .replace(r"\134", "\\") + .replace(r"\040", " ") + .replace(r"\011", "\t") + .replace(r"\012", "\n") +} + fn main() -> Result<()> { tracing_subscriber::fmt() .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .init(); let args = Args::parse(); - match args.platform { - TeePlatform::Tdx => run_backend::<tdx::TdxBackend>(args.mountpoint), + let config = load_config(&args.config)?; + let platform = args.platform.unwrap_or(config.platform); + fs_err::create_dir_all(&args.runtime_dir)?; + publish_trust_anchors(&args.runtime_dir, &config)?; + match platform { + TeeVariant::DstackTdx => { + simulate_dmi(&args.runtime_dir, &args.dmi_root, "Dstack", "dstack")?; + run_backend::<tdx::TdxBackend>(args.mountpoint, &config) + } + TeeVariant::DstackGcpTdx => { + simulate_dmi( + &args.runtime_dir, + &args.dmi_root, + "Google", + "Google Compute Engine", + )?; + tpm::start_gcp_vtpm(&args.runtime_dir, &config)?; + run_backend::<tdx::TdxBackend>(args.mountpoint, &config) + } + TeeVariant::DstackNitroEnclave => { + simulate_dmi( + &args.runtime_dir, + &args.dmi_root, + "AWS Nitro Enclaves", + "Nitro Enclave", + )?; + nsm::run(&config) + } + TeeVariant::DstackAwsNitroTpm => { + simulate_dmi(&args.runtime_dir, &args.dmi_root, "Amazon EC2", "t3.metal")?; + tpm::run_nitro_vtpm(&args.runtime_dir, &config) + } + TeeVariant::DstackAmdSevSnp => { + simulate_dmi(&args.runtime_dir, &args.dmi_root, "Dstack", "dstack")?; + run_backend::<sev_snp::SevSnpBackend>(args.mountpoint, &config) + } + } +} + +/// Publish the external trust anchors the guest verifier reads. +/// +/// The roots come from the same seed that signs the simulated evidence, so +/// nothing has to travel from the host: it supplies the seed for a fake TEE +/// device and never names a trust anchor. This binary ships only in the +/// development image, and the systemd unit orders it before `dstack-prepare`, +/// so `dstack-util` sees the roots when they exist and vendor production roots +/// everywhere else. +/// +/// Every platform's root is published regardless of the simulated platform: a +/// development guest also verifies a KMS and a gateway, and those need not run +/// on the platform this guest simulates. +fn publish_trust_anchors(runtime_dir: &Path, config: &TeeSimulatorConfig) -> Result<()> { + let seed = config + .mock_attestation_seed + .as_deref() + .context("tee_simulator.mock_attestation_seed is required")?; + let seed = mock_attestation::parse_seed(seed)?; + let base_url = config + .collateral_base_url + .as_deref() + .unwrap_or(DEFAULT_COLLATERAL_BASE_URL); + let pki = mock_attestation::server::MockCollateralState::from_seed(seed, base_url)?; + + let dir = trust_anchor_dir(runtime_dir); + fs_err::create_dir_all(&dir).context("failed to create the trust anchor directory")?; + // The reader refuses anything group- or world-writable, so the directory + // has to be tightened even when it already existed with a laxer mode. + fs_err::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)) + .context("failed to restrict the trust anchor directory")?; + let tdx = write_root(&dir, "tdx-root-ca.pem", &pki.tdx.root_ca_pem())?; + let tpm = write_root(&dir, "tpm-root-ca.pem", &pki.tpm.root_ca_pem())?; + let nsm = write_root(&dir, "nsm-root-ca.pem", &pki.nsm.root_ca_pem())?; + let sev_snp = write_root(&dir, "sev-snp-root-ca.pem", &pki.sev_snp.root_ca_pem())?; + + let root_ca = RootCaPaths { + tdx: Some(tdx), + gcp_tpm: Some(tpm.clone()), + aws_nitro_tpm: Some(tpm), + aws_nitro_enclave: Some(nsm), + sev_snp_milan: Some(sev_snp.clone()), + sev_snp_genoa: Some(sev_snp.clone()), + sev_snp_turin: Some(sev_snp), + }; + safe_write::safe_write_with_mode( + trust_anchors::roots_path(&dir), + serde_json::to_vec_pretty(&root_ca).context("failed to serialize published roots")?, + 0o600, + ) + .context("failed to write the published roots")?; + info!(dir = %dir.display(), "published external attestation trust anchors"); + Ok(()) +} + +fn write_root(dir: &Path, name: &str, pem: &str) -> Result<PathBuf> { + let path = dir.join(name); + safe_write::safe_write_with_mode(&path, pem.as_bytes(), 0o600) + .with_context(|| format!("failed to write {name}"))?; + Ok(path) +} + +fn trust_anchor_dir(runtime_dir: &Path) -> PathBuf { + runtime_dir.join(TRUST_ANCHOR_SUBDIR) +} + +/// Override the DMI strings exported by SeaBIOS through sysfs. Platform +/// detection intentionally remains unchanged and observes the same values as +/// it does on a real cloud VM. +fn simulate_dmi( + runtime_dir: &Path, + dmi_root: &Path, + sys_vendor: &str, + product_name: &str, +) -> Result<()> { + if dmi_root != Path::new("/sys/class/dmi/id") { + fs_err::create_dir_all(dmi_root)?; + fs_err::write(dmi_root.join("sys_vendor"), format!("{sys_vendor}\n"))?; + fs_err::write(dmi_root.join("product_name"), format!("{product_name}\n"))?; + return Ok(()); + } + let dmi_dir = runtime_dir.join("dmi"); + fs_err::create_dir_all(&dmi_dir)?; + for (name, value) in [("sys_vendor", sys_vendor), ("product_name", product_name)] { + let source = dmi_dir.join(name); + let target = dmi_root.join(name); + fs_err::write(&source, format!("{value}\n"))?; + let source = CString::new(source.as_os_str().as_bytes())?; + let target_c = CString::new(target.as_os_str().as_bytes())?; + let result = unsafe { + libc::mount( + source.as_ptr(), + target_c.as_ptr(), + std::ptr::null(), + libc::MS_BIND, + std::ptr::null(), + ) + }; + if result != 0 { + return Err(std::io::Error::last_os_error()).with_context(|| { + format!("failed to simulate SeaBIOS DMI file {}", target.display()) + }); + } } + Ok(()) +} + +fn load_config(path: &Path) -> Result<TeeSimulatorConfig> { + serde_json::from_slice( + &fs_err::read(path).with_context(|| format!("failed to read {}", path.display()))?, + ) + .with_context(|| format!("failed to parse {}", path.display())) } #[cfg(test)] @@ -99,7 +303,76 @@ mod tests { #[test] fn defaults_to_tdx() { let args = Args::try_parse_from(["dstack-tee-simulator"]).unwrap(); - assert_eq!(args.platform, TeePlatform::Tdx); + assert_eq!(args.platform, None); assert!(args.mountpoint.is_none()); } + + #[test] + fn missing_config_is_rejected() { + assert!(load_config(Path::new("/definitely/missing/config")).is_err()); + } + + #[test] + fn default_runtime_dir_matches_the_verifier_handoff_path() { + let args = Args::try_parse_from(["dstack-tee-simulator"]).unwrap(); + assert_eq!( + trust_anchor_dir(&args.runtime_dir), + Path::new(trust_anchors::ANCHOR_DIR) + ); + } + + #[test] + fn published_tdx_root_matches_the_seeded_pki() { + let dir = tempfile::tempdir().unwrap(); + let seed = [0x21; 32]; + publish_trust_anchors( + dir.path(), + &TeeSimulatorConfig { + mock_attestation_seed: Some(hex::encode(seed)), + ..Default::default() + }, + ) + .unwrap(); + + let roots = trust_anchors::load_anchors(&trust_anchor_dir(dir.path())) + .unwrap() + .expect("simulator should publish trust anchors"); + let published = fs_err::read(roots.tdx.as_ref().unwrap()).unwrap(); + let expected = mock_attestation::tdx::TdxGenerator::from_seed(seed) + .unwrap() + .root_ca_pem(); + assert_eq!(String::from_utf8(published).unwrap(), expected); + } + + #[test] + fn a_seedless_config_publishes_nothing() { + let dir = tempfile::tempdir().unwrap(); + assert!(publish_trust_anchors(dir.path(), &TeeSimulatorConfig::default()).is_err()); + assert!(trust_anchors::load_anchors(&trust_anchor_dir(dir.path())) + .unwrap() + .is_none()); + } + + #[test] + fn config_selects_each_platform() { + for (name, expected) in [ + ("dstack-tdx", TeeVariant::DstackTdx), + ("dstack-gcp-tdx", TeeVariant::DstackGcpTdx), + ("dstack-amd-sev-snp", TeeVariant::DstackAmdSevSnp), + ("dstack-nitro-enclave", TeeVariant::DstackNitroEnclave), + ("dstack-aws-nitro-tpm", TeeVariant::DstackAwsNitroTpm), + ] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sys-config.json"); + fs_err::write( + &path, + serde_json::json!({ + "platform": name + }) + .to_string(), + ) + .unwrap(); + assert_eq!(load_config(&path).unwrap().platform, expected); + } + } } diff --git a/dstack/tee-simulator/src/nsm.rs b/dstack/tee-simulator/src/nsm.rs new file mode 100644 index 000000000..b14c955ba --- /dev/null +++ b/dstack/tee-simulator/src/nsm.rs @@ -0,0 +1,561 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! CUSE implementation of the AWS Nitro Security Module ioctl ABI. + +use std::{ + collections::{BTreeMap, BTreeSet}, + ffi::CString, + mem, + os::raw::{c_int, c_uint, c_void}, + path::Path, + ptr, + sync::{Arc, Mutex, OnceLock}, + thread, + time::Duration, +}; + +use anyhow::{Context, Result}; +use aws_nitro_enclaves_nsm_api::api::{Digest, ErrorCode, Request, Response}; +use dstack_types::TeeSimulatorConfig; +use libc::{iovec, size_t}; +use mock_attestation::{nsm::NsmGenerator, parse_seed}; +use sha2::{Digest as _, Sha384}; + +static GENERATOR: OnceLock<Arc<NsmGenerator>> = OnceLock::new(); +static STATE: OnceLock<Mutex<NsmState>> = OnceLock::new(); +static CUSE: OnceLock<CuseApi> = OnceLock::new(); + +const PCR_COUNT: u16 = 32; +const PCR_SIZE: usize = 48; +const MAX_EXTEND_SIZE: usize = 1024; + +struct NsmState { + pcrs: Vec<[u8; PCR_SIZE]>, + locked: BTreeSet<u16>, +} + +impl Default for NsmState { + fn default() -> Self { + Self { + pcrs: vec![[0; PCR_SIZE]; PCR_COUNT.into()], + locked: BTreeSet::new(), + } + } +} + +impl NsmState { + fn measured() -> Self { + let mut state = Self::default(); + for index in 0..=2 { + let digest = Sha384::digest(format!("dstack-tee-simulator/nsm/pcr/{index}").as_bytes()); + state.pcrs[index].copy_from_slice(&digest); + } + state + } + + fn handle(&mut self, generator: &NsmGenerator, request: Request) -> Response { + match request { + Request::DescribePCR { index } => { + let Some(value) = self.pcrs.get(usize::from(index)) else { + return Response::Error(ErrorCode::InvalidIndex); + }; + Response::DescribePCR { + lock: self.locked.contains(&index), + data: value.to_vec(), + } + } + Request::ExtendPCR { index, data } => { + if data.len() > MAX_EXTEND_SIZE { + return Response::Error(ErrorCode::InputTooLarge); + } + let Some(value) = self.pcrs.get_mut(usize::from(index)) else { + return Response::Error(ErrorCode::InvalidIndex); + }; + if self.locked.contains(&index) { + return Response::Error(ErrorCode::ReadOnlyIndex); + } + let digest = Sha384::new() + .chain_update(value.as_slice()) + .chain_update(data) + .finalize(); + value.copy_from_slice(&digest); + Response::ExtendPCR { + data: value.to_vec(), + } + } + Request::LockPCR { index } => { + if index >= PCR_COUNT { + return Response::Error(ErrorCode::InvalidIndex); + } + self.locked.insert(index); + Response::LockPCR + } + Request::LockPCRs { range } => { + if range > PCR_COUNT { + return Response::Error(ErrorCode::InvalidIndex); + } + self.locked.extend(0..range); + Response::LockPCRs + } + Request::DescribeNSM => Response::DescribeNSM { + version_major: 1, + version_minor: 0, + version_patch: 0, + module_id: "dstack-tee-simulator".into(), + max_pcrs: PCR_COUNT, + locked_pcrs: self.locked.clone(), + digest: Digest::SHA384, + }, + Request::Attestation { + user_data, + nonce, + public_key, + } => { + let pcrs = self + .pcrs + .iter() + .enumerate() + .map(|(index, value)| (index as u16, value.to_vec())) + .collect::<BTreeMap<_, _>>(); + generator + .attest_with_claims( + user_data.as_ref().map(|value| value.as_slice()), + nonce.as_ref().map(|value| value.as_slice()), + public_key.as_ref().map(|value| value.as_slice()), + pcrs, + ) + .map(|document| Response::Attestation { document }) + .unwrap_or(Response::Error(ErrorCode::InternalError)) + } + Request::GetRandom => Response::GetRandom { + random: vec![0x42; 32], + }, + _ => Response::Error(ErrorCode::InvalidOperation), + } + } +} + +type FuseReq = *mut c_void; + +#[repr(C)] +struct FuseFileInfo { + _opaque: [u8; 0], +} + +#[repr(C)] +struct CuseInfo { + dev_major: c_uint, + dev_minor: c_uint, + dev_info_argc: c_uint, + dev_info_argv: *mut *const libc::c_char, + flags: c_uint, +} + +#[repr(C)] +struct CuseOperations { + init: *const c_void, + init_done: Option<unsafe extern "C" fn(*mut c_void)>, + destroy: *const c_void, + open: Option<unsafe extern "C" fn(FuseReq, *mut FuseFileInfo)>, + read: *const c_void, + write: *const c_void, + flush: *const c_void, + release: Option<unsafe extern "C" fn(FuseReq, *mut FuseFileInfo)>, + fsync: *const c_void, + ioctl: Option< + unsafe extern "C" fn( + FuseReq, + c_int, + *mut c_void, + *mut FuseFileInfo, + c_uint, + *const c_void, + size_t, + size_t, + ), + >, + poll: *const c_void, +} + +struct CuseApi { + main: unsafe extern "C" fn( + c_int, + *mut *mut libc::c_char, + *const CuseInfo, + *const CuseOperations, + *mut c_void, + ) -> c_int, + reply_open: unsafe extern "C" fn(FuseReq, *const FuseFileInfo) -> c_int, + reply_err: unsafe extern "C" fn(FuseReq, c_int) -> c_int, + reply_ioctl_retry: + unsafe extern "C" fn(FuseReq, *const iovec, size_t, *const iovec, size_t) -> c_int, + reply_ioctl: unsafe extern "C" fn(FuseReq, c_int, *const c_void, size_t) -> c_int, +} + +fn cuse() -> &'static CuseApi { + let Some(cuse) = CUSE.get() else { + std::process::abort(); + }; + cuse +} + +unsafe fn load_cuse() -> Result<CuseApi> { + // FUSE 3.18 bumped the shared-library SONAME to 4. Keep the older SONAME + // fallback so development binaries also run on distributions that still + // ship the previous ABI. + let library = libloading::Library::new("libfuse3.so.4") + .or_else(|_| libloading::Library::new("libfuse3.so.3"))?; + let library = Box::leak(Box::new(library)); + Ok(CuseApi { + main: *library.get(b"cuse_lowlevel_main\0")?, + reply_open: *library.get(b"fuse_reply_open\0")?, + reply_err: *library.get(b"fuse_reply_err\0")?, + reply_ioctl_retry: *library.get(b"fuse_reply_ioctl_retry\0")?, + reply_ioctl: *library.get(b"fuse_reply_ioctl\0")?, + }) +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct NsmMessage { + request: iovec, + response: iovec, +} + +unsafe extern "C" fn open(req: FuseReq, fi: *mut FuseFileInfo) { + (cuse().reply_open)(req, fi); +} + +unsafe extern "C" fn release(req: FuseReq, _fi: *mut FuseFileInfo) { + (cuse().reply_err)(req, 0); +} + +unsafe extern "C" fn init_done(_userdata: *mut c_void) { + if let Err(error) = ensure_nsm_device() { + tracing::error!(?error, "failed to publish simulated NSM device"); + return; + } + if let Err(error) = sd_notify::notify(true, &[sd_notify::NotifyState::Ready]) { + tracing::error!(?error, "failed to notify systemd that NSM is ready"); + } +} + +fn ensure_nsm_device() -> Result<()> { + let device_path = Path::new("/dev/nsm"); + let sysfs_paths = [ + Path::new("/sys/class/misc/nsm/dev"), + Path::new("/sys/devices/virtual/misc/nsm/dev"), + ]; + let mut stable_checks = 0; + for _ in 0..100 { + if device_path.exists() { + stable_checks += 1; + if stable_checks >= 10 { + return Ok(()); + } + thread::sleep(Duration::from_millis(20)); + continue; + } + stable_checks = 0; + if let Some(device) = sysfs_paths + .iter() + .find_map(|path| fs_err::read_to_string(path).ok()) + { + let (major, minor) = device + .trim() + .split_once(':') + .context("invalid NSM device number")?; + let major = major.parse::<u32>().context("invalid NSM major number")?; + let minor = minor.parse::<u32>().context("invalid NSM minor number")?; + let path = CString::new("/dev/nsm")?; + let result = unsafe { + libc::mknod( + path.as_ptr(), + libc::S_IFCHR | 0o660, + libc::makedev(major, minor), + ) + }; + if result == 0 || device_path.exists() { + thread::sleep(Duration::from_millis(20)); + continue; + } + return Err(std::io::Error::last_os_error()).context("failed to create /dev/nsm"); + } + thread::sleep(Duration::from_millis(20)); + } + anyhow::bail!("CUSE NSM device did not appear in sysfs") +} + +unsafe extern "C" fn ioctl( + req: FuseReq, + _cmd: c_int, + arg: *mut c_void, + _fi: *mut FuseFileInfo, + _flags: c_uint, + in_buf: *const c_void, + in_bufsz: size_t, + _out_bufsz: size_t, +) { + let message_size = mem::size_of::<NsmMessage>(); + if in_bufsz == 0 { + let message = iovec { + iov_base: arg, + iov_len: message_size, + }; + (cuse().reply_ioctl_retry)(req, &message, 1, &message, 1); + return; + } + if in_buf.is_null() || in_bufsz < message_size { + (cuse().reply_err)(req, libc::EINVAL); + return; + } + + let message = ptr::read_unaligned(in_buf.cast::<NsmMessage>()); + if in_bufsz == message_size { + if message.request.iov_len > 0x1000 || message.response.iov_len > 0x3000 { + (cuse().reply_err)(req, libc::EMSGSIZE); + return; + } + let input = [ + iovec { + iov_base: arg, + iov_len: message_size, + }, + message.request, + ]; + let output = [ + iovec { + iov_base: arg, + iov_len: message_size, + }, + message.response, + ]; + (cuse().reply_ioctl_retry)(req, input.as_ptr(), 2, output.as_ptr(), 2); + return; + } + + let request_bytes = std::slice::from_raw_parts( + in_buf.cast::<u8>().add(message_size), + in_bufsz - message_size, + ); + let response = match serde_cbor::from_slice::<Request>(request_bytes) { + Ok(request) => handle(request), + Err(_) => Response::Error(ErrorCode::InvalidOperation), + }; + let encoded = match serde_cbor::to_vec(&response) { + Ok(encoded) if encoded.len() <= message.response.iov_len => encoded, + _ => { + (cuse().reply_err)(req, libc::EMSGSIZE); + return; + } + }; + let mut updated = message; + updated.response.iov_len = encoded.len(); + let mut output = Vec::with_capacity(message_size + encoded.len()); + output.extend_from_slice(std::slice::from_raw_parts( + (&updated as *const NsmMessage).cast::<u8>(), + message_size, + )); + output.extend_from_slice(&encoded); + (cuse().reply_ioctl)(req, 0, output.as_ptr().cast(), output.len()); +} + +fn handle(request: Request) -> Response { + let Some(generator) = GENERATOR.get() else { + return Response::Error(ErrorCode::InternalError); + }; + let Some(state) = STATE.get() else { + return Response::Error(ErrorCode::InternalError); + }; + state + .lock() + .map(|mut state| state.handle(generator, request)) + .unwrap_or(Response::Error(ErrorCode::InternalError)) +} + +pub fn run(config: &TeeSimulatorConfig) -> Result<()> { + anyhow::ensure!( + !std::path::Path::new("/dev/nsm").exists(), + "refusing to replace a real NSM device" + ); + CUSE.set(unsafe { load_cuse()? }) + .map_err(|_| anyhow::anyhow!("CUSE API was already initialized"))?; + let seed = config + .mock_attestation_seed + .as_deref() + .context("tee_simulator.mock_attestation_seed is required")?; + GENERATOR + .set(Arc::new(NsmGenerator::from_seed(parse_seed(seed)?)?)) + .map_err(|_| anyhow::anyhow!("NSM simulator was already initialized"))?; + STATE + .set(Mutex::new(NsmState::measured())) + .map_err(|_| anyhow::anyhow!("NSM simulator state was already initialized"))?; + + let device_name = CString::new("DEVNAME=nsm")?; + let mut device_args = [device_name.as_ptr(), ptr::null()]; + let info = CuseInfo { + dev_major: 0, + dev_minor: 0, + dev_info_argc: 1, + dev_info_argv: device_args.as_mut_ptr(), + flags: 1, // CUSE_UNRESTRICTED_IOCTL + }; + let operations = CuseOperations { + init: ptr::null(), + init_done: Some(init_done), + destroy: ptr::null(), + open: Some(open), + read: ptr::null(), + write: ptr::null(), + flush: ptr::null(), + release: Some(release), + fsync: ptr::null(), + ioctl: Some(ioctl), + poll: ptr::null(), + }; + let program = CString::new("dstack-tee-simulator")?; + let foreground = CString::new("-f")?; + let single_threaded = CString::new("-s")?; + let mut argv = [ + program.as_ptr().cast_mut(), + foreground.as_ptr().cast_mut(), + single_threaded.as_ptr().cast_mut(), + ]; + let result = unsafe { + (cuse().main)( + argv.len() as c_int, + argv.as_mut_ptr(), + &info, + &operations, + ptr::null_mut(), + ) + }; + anyhow::ensure!(result == 0, "CUSE NSM session failed with status {result}"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use aws_nitro_enclaves_nsm_api::api::Request; + + fn response_kind(response: &Response) -> &'static str { + match response { + Response::DescribePCR { .. } => "DescribePCR", + Response::ExtendPCR { .. } => "ExtendPCR", + Response::LockPCR => "LockPCR", + Response::LockPCRs => "LockPCRs", + Response::DescribeNSM { .. } => "DescribeNSM", + Response::Attestation { .. } => "Attestation", + Response::GetRandom { .. } => "GetRandom", + Response::Error(_) => "Error", + _ => "Unknown", + } + } + + #[test] + fn measured_state_models_a_production_enclave() { + let state = NsmState::measured(); + assert!(state.pcrs[..=2] + .iter() + .all(|value| value.iter().any(|byte| *byte != 0))); + assert!(state.pcrs[3..] + .iter() + .all(|value| value.iter().all(|byte| *byte == 0))); + } + + #[test] + fn pcr_lifecycle_matches_nsm_semantics() { + let generator = NsmGenerator::from_seed([0x51; 32]).unwrap(); + let mut state = NsmState::default(); + match state.handle(&generator, Request::DescribePCR { index: 0 }) { + Response::DescribePCR { lock, data } => { + assert!(!lock); + assert_eq!(data, vec![0; PCR_SIZE]); + } + response => panic!("unexpected {}", response_kind(&response)), + } + let input = b"measurement".to_vec(); + let expected = Sha384::new() + .chain_update([0; PCR_SIZE]) + .chain_update(&input) + .finalize() + .to_vec(); + match state.handle( + &generator, + Request::ExtendPCR { + index: 0, + data: input, + }, + ) { + Response::ExtendPCR { data } => assert_eq!(data, expected), + response => panic!("unexpected {}", response_kind(&response)), + } + assert!(matches!( + state.handle(&generator, Request::LockPCR { index: 0 }), + Response::LockPCR + )); + assert!(matches!( + state.handle( + &generator, + Request::ExtendPCR { + index: 0, + data: vec![1], + }, + ), + Response::Error(ErrorCode::ReadOnlyIndex) + )); + assert!(matches!( + state.handle(&generator, Request::DescribePCR { index: PCR_COUNT }), + Response::Error(ErrorCode::InvalidIndex) + )); + assert!(matches!( + state.handle( + &generator, + Request::ExtendPCR { + index: 1, + data: vec![0; MAX_EXTEND_SIZE + 1], + }, + ), + Response::Error(ErrorCode::InputTooLarge) + )); + } + + #[test] + fn attestation_binds_claims_and_current_pcrs() { + let generator = NsmGenerator::from_seed([0x52; 32]).unwrap(); + let verifier = nsm_qvl::QuoteVerifier::new(generator.root_ca_pem()); + let mut state = NsmState::default(); + assert!(matches!( + state.handle( + &generator, + Request::ExtendPCR { + index: 2, + data: b"app".to_vec(), + }, + ), + Response::ExtendPCR { .. } + )); + let response = state.handle( + &generator, + Request::Attestation { + user_data: Some(b"user".to_vec().into()), + nonce: Some(b"nonce".to_vec().into()), + public_key: Some(b"public".to_vec().into()), + }, + ); + let Response::Attestation { document } = response else { + panic!("unexpected {}", response_kind(&response)); + }; + let verified = verifier.verify(&document, None, None).unwrap(); + assert_eq!(verified.user_data.as_deref(), Some(b"user".as_slice())); + assert_eq!(verified.nonce.as_deref(), Some(b"nonce".as_slice())); + assert_eq!(verified.public_key.as_deref(), Some(b"public".as_slice())); + assert_eq!( + verified.pcrs.get(&2).map(Vec::as_slice), + Some(state.pcrs[2].as_slice()) + ); + } +} diff --git a/dstack/tee-simulator/src/sev_snp.rs b/dstack/tee-simulator/src/sev_snp.rs new file mode 100644 index 000000000..0d58ac1a1 --- /dev/null +++ b/dstack/tee-simulator/src/sev_snp.rs @@ -0,0 +1,420 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + ffi::OsStr, + path::Path, + sync::Arc, + time::{Duration, SystemTime}, +}; + +use anyhow::{Context, Result}; +use dstack_types::TeeSimulatorConfig; +use fuser::{ + FileAttr, FileType, Filesystem, ReplyAttr, ReplyCreate, ReplyData, ReplyDirectory, ReplyEmpty, + ReplyEntry, ReplyOpen, ReplyWrite, Request, +}; +use mock_attestation::sev_snp::SevSnpGenerator; + +use crate::TeeBackend; + +const ROOT: u64 = 1; +const ENTRY: u64 = 2; +const PROVIDER: u64 = 3; +const INBLOB: u64 = 4; +const OUTBLOB: u64 = 5; +const CERTS: u64 = 6; +const TTL: Duration = Duration::ZERO; + +pub(crate) struct SevSnpFs { + generator: Arc<SevSnpGenerator>, + report: Vec<u8>, + certs: Vec<u8>, + uid: u32, + gid: u32, + host_data: [u8; 32], +} + +impl SevSnpFs { + fn new( + generator: Arc<SevSnpGenerator>, + host_data: [u8; 32], + measurement: [u8; 48], + ) -> Result<Self> { + let evidence = generator.attest_with_measurement([0; 64], host_data, measurement)?; + Ok(Self { + generator, + report: evidence.report, + certs: encode_cert_table(&evidence.cert_chain)?, + uid: unsafe { libc::geteuid() }, + gid: unsafe { libc::getegid() }, + host_data, + }) + } + + fn attr(&self, ino: u64) -> Option<FileAttr> { + let (kind, perm, size) = match ino { + ROOT | ENTRY => (FileType::Directory, 0o755, 0), + PROVIDER => (FileType::RegularFile, 0o444, 10), + INBLOB => (FileType::RegularFile, 0o200, 0), + OUTBLOB => (FileType::RegularFile, 0o400, self.report.len() as u64), + CERTS => (FileType::RegularFile, 0o400, self.certs.len() as u64), + _ => return None, + }; + Some(FileAttr { + ino, + size, + blocks: 0, + atime: SystemTime::UNIX_EPOCH, + mtime: SystemTime::UNIX_EPOCH, + ctime: SystemTime::UNIX_EPOCH, + crtime: SystemTime::UNIX_EPOCH, + kind, + perm, + nlink: if kind == FileType::Directory { 2 } else { 1 }, + uid: self.uid, + gid: self.gid, + rdev: 0, + blksize: 4096, + flags: 0, + }) + } + + fn child(parent: u64, name: &OsStr) -> Option<u64> { + match (parent, name.to_str()?) { + (ROOT, "provider") | (ENTRY, "provider") => Some(PROVIDER), + (ENTRY, "inblob" | "reportdata" | "report_data") => Some(INBLOB), + (ENTRY, "outblob" | "report") => Some(OUTBLOB), + (ENTRY, "certs" | "cert_chain" | "auxblob") => Some(CERTS), + (ROOT, name) if name.starts_with("dstack-") => Some(ENTRY), + _ => None, + } + } + + fn update_report(&mut self, data: &[u8]) -> Result<()> { + let report_data: [u8; 64] = data + .try_into() + .context("SEV-SNP report data must be exactly 64 bytes")?; + let measurement: [u8; 48] = self + .report + .get(0x90..0xc0) + .context("SEV-SNP report does not contain a measurement")? + .try_into() + .context("SEV-SNP measurement has an invalid length")?; + let evidence = + self.generator + .attest_with_measurement(report_data, self.host_data, measurement)?; + let certs = encode_cert_table(&evidence.cert_chain)?; + self.report = evidence.report; + self.certs = certs; + Ok(()) + } +} + +fn bounded_read(data: &[u8], offset: i64, size: u32) -> Result<&[u8], i32> { + if offset < 0 { + return Err(libc::EINVAL); + } + let start = offset as usize; + Ok(data + .get(start..start.saturating_add(size as usize).min(data.len())) + .unwrap_or_default()) +} + +impl Filesystem for SevSnpFs { + fn lookup(&mut self, _req: &Request<'_>, parent: u64, name: &OsStr, reply: ReplyEntry) { + match Self::child(parent, name).and_then(|ino| self.attr(ino).map(|attr| (ino, attr))) { + Some((_, attr)) => reply.entry(&TTL, &attr, 0), + None => reply.error(libc::ENOENT), + } + } + fn getattr(&mut self, _req: &Request<'_>, ino: u64, _fh: Option<u64>, reply: ReplyAttr) { + match self.attr(ino) { + Some(attr) => reply.attr(&TTL, &attr), + None => reply.error(libc::ENOENT), + } + } + fn mkdir( + &mut self, + _req: &Request<'_>, + parent: u64, + name: &OsStr, + _mode: u32, + _umask: u32, + reply: ReplyEntry, + ) { + if parent == ROOT + && name + .to_str() + .is_some_and(|name| name.starts_with("dstack-")) + { + match self.attr(ENTRY) { + Some(attr) => reply.entry(&TTL, &attr, 0), + None => reply.error(libc::EIO), + } + } else { + reply.error(libc::EACCES); + } + } + fn rmdir(&mut self, _req: &Request<'_>, parent: u64, _name: &OsStr, reply: ReplyEmpty) { + if parent == ROOT { + reply.ok(); + } else { + reply.error(libc::EACCES); + } + } + fn readdir( + &mut self, + _req: &Request<'_>, + ino: u64, + _fh: u64, + offset: i64, + mut reply: ReplyDirectory, + ) { + let entries: &[(u64, FileType, &str)] = match ino { + ROOT => &[ + (ROOT, FileType::Directory, "."), + (ROOT, FileType::Directory, ".."), + (PROVIDER, FileType::RegularFile, "provider"), + ], + ENTRY => &[ + (ENTRY, FileType::Directory, "."), + (ROOT, FileType::Directory, ".."), + (PROVIDER, FileType::RegularFile, "provider"), + (INBLOB, FileType::RegularFile, "inblob"), + (OUTBLOB, FileType::RegularFile, "outblob"), + (CERTS, FileType::RegularFile, "certs"), + ], + _ => { + reply.error(libc::ENOENT); + return; + } + }; + for (index, entry) in entries.iter().enumerate().skip(offset.max(0) as usize) { + if reply.add(entry.0, (index + 1) as i64, entry.1, entry.2) { + break; + } + } + reply.ok(); + } + fn open(&mut self, _req: &Request<'_>, ino: u64, _flags: i32, reply: ReplyOpen) { + if matches!(ino, PROVIDER | INBLOB | OUTBLOB | CERTS) { + reply.opened(0, 1); + } else { + reply.error(libc::ENOENT); + } + } + fn create( + &mut self, + _req: &Request<'_>, + parent: u64, + name: &OsStr, + _mode: u32, + _umask: u32, + _flags: i32, + reply: ReplyCreate, + ) { + match Self::child(parent, name).filter(|ino| *ino == INBLOB) { + Some(ino) => match self.attr(ino) { + Some(attr) => reply.created(&TTL, &attr, 0, 0, 1), + None => reply.error(libc::EIO), + }, + None => reply.error(libc::EACCES), + } + } + fn read( + &mut self, + _req: &Request<'_>, + ino: u64, + _fh: u64, + offset: i64, + size: u32, + _flags: i32, + _lock: Option<u64>, + reply: ReplyData, + ) { + let data: &[u8] = match ino { + PROVIDER => b"sev_guest\n", + OUTBLOB => &self.report, + CERTS => &self.certs, + _ => { + reply.error(libc::EACCES); + return; + } + }; + match bounded_read(data, offset, size) { + Ok(data) => reply.data(data), + Err(errno) => reply.error(errno), + } + } + fn write( + &mut self, + _req: &Request<'_>, + ino: u64, + _fh: u64, + offset: i64, + data: &[u8], + _write_flags: u32, + _flags: i32, + _lock: Option<u64>, + reply: ReplyWrite, + ) { + if ino != INBLOB || offset != 0 { + reply.error(libc::EINVAL); + return; + } + match self.update_report(data) { + Ok(()) => reply.written(data.len() as u32), + Err(_) if data.len() != 64 => reply.error(libc::EINVAL), + Err(_) => reply.error(libc::EIO), + } + } +} + +fn encode_cert_table(cert_chain: &[Vec<u8>]) -> Result<Vec<u8>> { + const ASK_GUID: [u8; 16] = [ + 0x4a, 0xb7, 0xb3, 0x79, 0xbb, 0xac, 0x4f, 0xe4, 0xa0, 0x2f, 0x05, 0xae, 0xf3, 0x27, 0xc7, + 0x82, + ]; + const VCEK_GUID: [u8; 16] = [ + 0x63, 0xda, 0x75, 0x8d, 0xe6, 0x64, 0x45, 0x64, 0xad, 0xc5, 0xf4, 0xb9, 0x3b, 0xe8, 0xac, + 0xcd, + ]; + let [ask, vcek] = cert_chain else { + anyhow::bail!("mock SEV-SNP evidence must contain ASK and VCEK"); + }; + let ask = pem::parse(ask)?.into_contents(); + let vcek = pem::parse(vcek)?.into_contents(); + let header_len = 3 * 24; + let mut output = Vec::with_capacity(header_len + ask.len() + vcek.len()); + for (guid, offset, length) in [ + (ASK_GUID, header_len, ask.len()), + (VCEK_GUID, header_len + ask.len(), vcek.len()), + ] { + output.extend_from_slice(&guid); + output.extend_from_slice(&(offset as u32).to_le_bytes()); + output.extend_from_slice(&(length as u32).to_le_bytes()); + } + output.extend_from_slice(&[0u8; 24]); + output.extend_from_slice(&ask); + output.extend_from_slice(&vcek); + Ok(output) +} + +pub(crate) struct SevSnpBackend; +impl TeeBackend for SevSnpBackend { + type Fs = SevSnpFs; + const PLATFORM: &'static str = "amd-sev-snp"; + const DEFAULT_MOUNTPOINT: &'static str = "/sys/kernel/config/tsm/report"; + fn create_filesystem(config: &TeeSimulatorConfig) -> Result<Self::Fs> { + let seed = config + .mock_attestation_seed + .as_deref() + .context("tee_simulator.mock_attestation_seed is required")?; + let mr_config = config + .mr_config + .as_deref() + .context("mr_config is required for SEV-SNP simulation")?; + let host_data = dstack_types::mr_config::MrConfigV3::snp_host_data_from_document(mr_config); + let measurement = match config.vm_config.as_deref() { + Some(vm_config) + if serde_json::from_str::<serde_json::Value>(vm_config)? + .get("sev_snp_measurement") + .is_some() => + { + let inputs = dstack_mr::sev::parse_snp_inputs_from_vm_config(vm_config)?; + dstack_mr::sev::compute_expected_measurement(&inputs.input)? + } + _ => [0x33; 48], + }; + SevSnpFs::new( + Arc::new(SevSnpGenerator::from_seed(mock_attestation::parse_seed( + seed, + )?)?), + host_data, + measurement, + ) + } + fn prepare_mountpoint(mountpoint: &Path) -> Result<()> { + crate::tdx::ensure_configfs_mount(mountpoint) + } + fn real_tee_available() -> bool { + Path::new("/dev/sev-guest").exists() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture() -> (Arc<SevSnpGenerator>, SevSnpFs) { + let generator = Arc::new(SevSnpGenerator::from_seed([0x42; 32]).unwrap()); + let fs = SevSnpFs::new(generator.clone(), [0x22; 32], [0x33; 48]).unwrap(); + (generator, fs) + } + + #[test] + fn report_update_is_verified_and_failure_atomic() { + let (generator, mut fs) = fixture(); + let original_report = fs.report.clone(); + let original_certs = fs.certs.clone(); + assert!(fs.update_report(&[0x11; 63]).is_err()); + assert_eq!(fs.report, original_report); + assert_eq!(fs.certs, original_certs); + + let report_data = [0x55; 64]; + fs.update_report(&report_data).unwrap(); + let evidence = generator + .attest_with_measurement(report_data, [0x22; 32], [0x33; 48]) + .unwrap(); + let verifier = sev_snp_qvl::QuoteVerifier::new( + generator.root_ca_pem().into_bytes(), + generator.root_ca_pem().into_bytes(), + generator.root_ca_pem().into_bytes(), + ); + verifier + .verify(&fs.report, &evidence.cert_chain, &report_data) + .unwrap(); + + let first_report = fs.report.clone(); + fs.update_report(&[0x77; 64]).unwrap(); + assert_ne!(fs.report, first_report); + } + + #[test] + fn filesystem_aliases_and_permissions_are_bounded() { + let (_, fs) = fixture(); + for alias in ["inblob", "reportdata", "report_data"] { + assert_eq!(SevSnpFs::child(ENTRY, OsStr::new(alias)), Some(INBLOB)); + } + for alias in ["outblob", "report"] { + assert_eq!(SevSnpFs::child(ENTRY, OsStr::new(alias)), Some(OUTBLOB)); + } + for alias in ["certs", "cert_chain", "auxblob"] { + assert_eq!(SevSnpFs::child(ENTRY, OsStr::new(alias)), Some(CERTS)); + } + assert_eq!(fs.attr(INBLOB).unwrap().perm, 0o200); + assert_eq!(fs.attr(OUTBLOB).unwrap().perm, 0o400); + assert_eq!(fs.attr(CERTS).unwrap().perm, 0o400); + assert_eq!(bounded_read(&fs.report, -1, 1), Err(libc::EINVAL)); + assert_eq!(bounded_read(&fs.report, 0, 8).unwrap(), &fs.report[..8]); + assert!(bounded_read(&fs.report, i64::MAX, u32::MAX) + .unwrap() + .is_empty()); + assert_eq!(SevSnpFs::child(ENTRY, OsStr::new("unknown")), None); + } + + #[test] + fn malformed_certificate_chains_fail_closed() { + assert!(encode_cert_table(&[]).is_err()); + assert!( + encode_cert_table(&[b"not a certificate".to_vec(), b"also invalid".to_vec()]).is_err() + ); + assert!(encode_cert_table(&[ + b"-----BEGIN CERTIFICATE-----\ninvalid\n-----END CERTIFICATE-----".to_vec(), + b"-----BEGIN CERTIFICATE-----\ninvalid\n-----END CERTIFICATE-----".to_vec(), + ]) + .is_err()); + } +} diff --git a/dstack/tee-simulator/src/tdx.rs b/dstack/tee-simulator/src/tdx.rs index 60e0603dd..f6bb4b5f3 100644 --- a/dstack/tee-simulator/src/tdx.rs +++ b/dstack/tee-simulator/src/tdx.rs @@ -2,15 +2,18 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::ffi::{CString, OsStr}; +use std::ffi::OsStr; use std::path::Path; +use std::sync::Arc; use std::time::{Duration, SystemTime}; use anyhow::{bail, Context, Result}; +use dstack_types::{TeeSimulatorConfig, VmConfig}; use fuser::{ FileAttr, FileType, Filesystem, ReplyAttr, ReplyData, ReplyDirectory, ReplyEmpty, ReplyEntry, ReplyOpen, ReplyWrite, Request, TimeOrNow, }; +use mock_attestation::tdx::TdxGenerator; use sha2::{Digest, Sha384}; use crate::TeeBackend; @@ -31,17 +34,15 @@ const MEASUREMENTS_DIR_INO: u64 = 7; const RTMR0_INO: u64 = 8; const CCEL_INO: u64 = 12; -const TDX_QUOTE_MIN_SIZE: usize = 632; const MR_CONFIG_ID_RANGE: std::ops::Range<usize> = 232..280; const RTMR0_OFFSET: usize = 376; const REPORT_DATA_RANGE: std::ops::Range<usize> = 568..632; -const QUOTE_FIXTURE: &[u8] = include_bytes!("../../ra-tls/assets/tdx_quote"); const CCEL_FIXTURE: &[u8] = include_bytes!("../../cc-eventlog/samples/ccel.bin"); -#[derive(Clone)] struct SimulatorState { - base_quote: Vec<u8>, + generator: Arc<TdxGenerator>, + mrtd: [u8; 48], ccel: Vec<u8>, rtmrs: [[u8; 48]; 4], outblob: Vec<u8>, @@ -49,42 +50,46 @@ struct SimulatorState { } impl SimulatorState { - fn new(base_quote: &[u8], ccel: &[u8]) -> Result<Self> { - if base_quote.len() < TDX_QUOTE_MIN_SIZE { - bail!("tdx quote fixture is too short: {}", base_quote.len()); - } - + fn new(generator: Arc<TdxGenerator>, ccel: &[u8], vm_config: Option<&str>) -> Result<Self> { + let ccel = ccel_for_config(ccel, vm_config)?; + let (mrtd, rtmrs) = measurements_for_config(&ccel, vm_config)?; let mut state = Self { - base_quote: base_quote.to_vec(), - ccel: ccel.to_vec(), - rtmrs: replay_boot_rtmrs(ccel)?, + generator, + mrtd, + ccel, + rtmrs, outblob: Vec::new(), generation: 0, }; - state.outblob = state.make_quote([0u8; 64]); + state.outblob = state.make_quote([0u8; 64])?; Ok(state) } - fn make_quote(&self, report_data: [u8; 64]) -> Vec<u8> { - let mut quote = self.base_quote.clone(); + fn make_quote(&self, report_data: [u8; 64]) -> Result<Vec<u8>> { + let mut quote = self + .generator + .attest_with_measurements(report_data, self.mrtd, self.rtmrs)? + .quote; quote[MR_CONFIG_ID_RANGE].fill(0); for (index, rtmr) in self.rtmrs.iter().enumerate() { let start = RTMR0_OFFSET + index * 48; quote[start..start + 48].copy_from_slice(rtmr); } quote[REPORT_DATA_RANGE].copy_from_slice(&report_data); - quote + Ok(quote) } fn request_quote(&mut self, report_data: &[u8]) -> Result<()> { let report_data: [u8; 64] = report_data .try_into() .map_err(|_| anyhow::anyhow!("inblob must be exactly 64 bytes"))?; - self.outblob = self.make_quote(report_data); - self.generation = self + let generation = self .generation .checked_add(1) .context("tsm generation overflow")?; + let outblob = self.make_quote(report_data)?; + self.outblob = outblob; + self.generation = generation; Ok(()) } @@ -115,6 +120,114 @@ impl SimulatorState { } } +fn ccel_for_config(ccel: &[u8], vm_config: Option<&str>) -> Result<Vec<u8>> { + let Some(vm_config) = vm_config else { + return Ok(ccel.to_vec()); + }; + let vm_config: VmConfig = serde_json::from_str(vm_config).context("invalid TDX vm_config")?; + let Some(document) = vm_config.tdx_measurement.as_ref() else { + return Ok(ccel.to_vec()); + }; + let measurement = document + .decode_measurement() + .map_err(anyhow::Error::msg) + .context("invalid TDX measurement document")?; + let expected = + dstack_mr::tdx::expected_rtmr0_acpi_hashes(&vm_config, measurement.tdvf.ovmf_variant) + .context("failed to generate TDX simulator ACPI measurements")?; + let events = cc_eventlog::tdx::decode_ccel(ccel).context("failed to decode CCEL fixture")?; + let mut patched = ccel.to_vec(); + + for (name, expected_digest) in [ + (cc_eventlog::tdx::TDX_ACPI_LOADER_EVENT, expected.loader), + (cc_eventlog::tdx::TDX_ACPI_RSDP_EVENT, expected.rsdp), + (cc_eventlog::tdx::TDX_ACPI_TABLES_EVENT, expected.tables), + ] { + let current = events + .iter() + .find(|event| event.imr == 0 && event.event == name) + .with_context(|| format!("CCEL fixture is missing {name}"))? + .digest(); + if current.len() != expected_digest.len() { + bail!( + "CCEL fixture {name} digest has length {}, expected {}", + current.len(), + expected_digest.len() + ); + } + let offsets = patched + .windows(current.len()) + .enumerate() + .filter_map(|(offset, bytes)| (bytes == current).then_some(offset)) + .collect::<Vec<_>>(); + let [offset] = offsets.as_slice() else { + bail!( + "CCEL fixture contains {} copies of the {name} digest, expected one", + offsets.len() + ); + }; + patched[*offset..*offset + current.len()].copy_from_slice(&expected_digest); + } + Ok(patched) +} + +fn measurements_for_config( + ccel: &[u8], + vm_config: Option<&str>, +) -> Result<([u8; 48], [[u8; 48]; 4])> { + let mut rtmrs = replay_boot_rtmrs(ccel)?; + let Some(vm_config) = vm_config else { + return Ok(([0x11; 48], rtmrs)); + }; + let value: serde_json::Value = serde_json::from_str(vm_config).context("invalid vm_config")?; + if value.get("tdx_measurement").is_none() { + return Ok(([0x11; 48], rtmrs)); + } + let vm_config: VmConfig = serde_json::from_value(value).context("invalid TDX vm_config")?; + let Some(document) = vm_config.tdx_measurement.as_ref() else { + return Ok(([0x11; 48], rtmrs)); + }; + let events = cc_eventlog::tdx::decode_ccel(ccel).context("failed to decode CCEL fixture")?; + let digest = |name: &str| -> Result<Vec<u8>> { + let event = events + .iter() + .find(|event| event.imr == 0 && event.event == name) + .with_context(|| format!("CCEL fixture is missing {name}"))?; + Ok(event.digest()) + }; + let acpi_hashes = dstack_mr::tdx::TdxRtmr0AcpiHashes { + loader: digest("acpi-loader")?, + rsdp: digest("acpi-rsdp")?, + tables: digest("acpi-tables")?, + }; + let measurements = dstack_mr::tdx::tdx_measurements_from_measurement_document( + document, + &vm_config, + &acpi_hashes, + )?; + let mrtd = measurements + .mrtd + .as_slice() + .try_into() + .context("invalid MRTD")?; + rtmrs[0] = measurements + .rtmr0 + .as_slice() + .try_into() + .context("invalid RTMR0")?; + rtmrs[1] = measurements + .rtmr1 + .as_slice() + .try_into() + .context("invalid RTMR1")?; + rtmrs[2] = measurements + .rtmr2 + .as_slice() + .try_into() + .context("invalid RTMR2")?; + Ok((mrtd, rtmrs)) +} + fn replay_boot_rtmrs(ccel: &[u8]) -> Result<[[u8; 48]; 4]> { let events = cc_eventlog::tdx::decode_ccel(ccel).context("failed to decode CCEL fixture")?; let mut rtmrs = [[0u8; 48]; 4]; @@ -145,11 +258,11 @@ pub(crate) struct TdxSimulatorFs { } impl TdxSimulatorFs { - fn new() -> Result<Self> { + fn new(generator: Arc<TdxGenerator>) -> Result<Self> { Ok(Self { - state: SimulatorState::new(QUOTE_FIXTURE, CCEL_FIXTURE)?, - uid: unsafe { libc::geteuid() }, - gid: unsafe { libc::getegid() }, + state: SimulatorState::new(generator, CCEL_FIXTURE, None)?, + uid: nix::unistd::geteuid().as_raw(), + gid: nix::unistd::getegid().as_raw(), }) } @@ -381,7 +494,7 @@ impl Filesystem for TdxSimulatorFs { } } -fn ensure_configfs_mount(mountpoint: &Path) -> Result<()> { +pub(crate) fn ensure_configfs_mount(mountpoint: &Path) -> Result<()> { if mountpoint != Path::new(TDX_DEFAULT_MOUNTPOINT) { std::fs::create_dir_all(mountpoint) .with_context(|| format!("failed to create {}", mountpoint.display()))?; @@ -389,25 +502,44 @@ fn ensure_configfs_mount(mountpoint: &Path) -> Result<()> { } if !mountpoint.is_dir() { - let source = CString::new("configfs")?; - let target = CString::new("/sys/kernel/config")?; - let fstype = CString::new("configfs")?; - let rc = unsafe { - libc::mount( - source.as_ptr(), - target.as_ptr(), - fstype.as_ptr(), - libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC, - std::ptr::null(), - ) - }; - if rc != 0 { - let error = std::io::Error::last_os_error(); - if error.raw_os_error() != Some(libc::EBUSY) { + let flags = nix::mount::MsFlags::MS_NOSUID + | nix::mount::MsFlags::MS_NODEV + | nix::mount::MsFlags::MS_NOEXEC; + if let Err(error) = nix::mount::mount( + Some("configfs"), + "/sys/kernel/config", + Some("configfs"), + flags, + None::<&str>, + ) { + if error != nix::errno::Errno::EBUSY { return Err(error).context("failed to mount configfs"); } } } + // configfs rejects arbitrary directories when no kernel TSM provider has + // registered the `tsm` subsystem. In a no-TEE development guest the + // simulator is that provider, so shadow the otherwise-empty configfs with + // a private tmpfs and create the userspace ABI hierarchy there. + if let Err(error) = std::fs::create_dir_all(mountpoint) { + if !matches!(error.raw_os_error(), Some(libc::EPERM) | Some(libc::EACCES)) { + return Err(error) + .with_context(|| format!("failed to create {}", mountpoint.display())); + } + let flags = nix::mount::MsFlags::MS_NOSUID + | nix::mount::MsFlags::MS_NODEV + | nix::mount::MsFlags::MS_NOEXEC; + nix::mount::mount( + Some("dstack-tee-simulator"), + "/sys/kernel/config", + Some("tmpfs"), + flags, + Some("mode=0755"), + ) + .context("failed to mount simulator configfs shadow")?; + std::fs::create_dir_all(mountpoint) + .with_context(|| format!("failed to create {}", mountpoint.display()))?; + } if !mountpoint.is_dir() { bail!( "tsm report mountpoint is unavailable: {}", @@ -432,8 +564,22 @@ impl TeeBackend for TdxBackend { const PLATFORM: &'static str = "tdx"; const DEFAULT_MOUNTPOINT: &'static str = TDX_DEFAULT_MOUNTPOINT; - fn create_filesystem() -> Result<Self::Fs> { - TdxSimulatorFs::new() + fn create_filesystem(config: &TeeSimulatorConfig) -> Result<Self::Fs> { + let seed = config + .mock_attestation_seed + .as_deref() + .context("tee_simulator.mock_attestation_seed is required")?; + let generator = Arc::new(TdxGenerator::from_seed(mock_attestation::parse_seed( + seed, + )?)?); + let mut fs = TdxSimulatorFs::new(generator)?; + let ccel = ccel_for_config(CCEL_FIXTURE, config.vm_config.as_deref())?; + let (mrtd, rtmrs) = measurements_for_config(&ccel, config.vm_config.as_deref())?; + fs.state.ccel = ccel; + fs.state.mrtd = mrtd; + fs.state.rtmrs = rtmrs; + fs.state.outblob = fs.state.make_quote([0; 64])?; + Ok(fs) } fn prepare_mountpoint(mountpoint: &Path) -> Result<()> { @@ -450,9 +596,48 @@ mod tests { use super::*; use dcap_qvl::quote::Quote; + #[test] + fn configured_ccel_uses_generated_acpi_digests() { + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../verifier/fixtures/tdx-lite-getquote.json" + )) + .unwrap(); + let vm_config_json = fixture["vm_config"].as_str().unwrap(); + let vm_config: VmConfig = serde_json::from_str(vm_config_json).unwrap(); + let measurement = vm_config + .tdx_measurement + .as_ref() + .unwrap() + .decode_measurement() + .unwrap(); + let expected = + dstack_mr::tdx::expected_rtmr0_acpi_hashes(&vm_config, measurement.tdvf.ovmf_variant) + .unwrap(); + + let patched = ccel_for_config(CCEL_FIXTURE, Some(vm_config_json)).unwrap(); + let events = cc_eventlog::tdx::decode_ccel(&patched).unwrap(); + for (name, digest) in [ + (cc_eventlog::tdx::TDX_ACPI_LOADER_EVENT, expected.loader), + (cc_eventlog::tdx::TDX_ACPI_RSDP_EVENT, expected.rsdp), + (cc_eventlog::tdx::TDX_ACPI_TABLES_EVENT, expected.tables), + ] { + let event = events + .iter() + .find(|event| event.imr == 0 && event.event == name) + .unwrap(); + assert_eq!(event.digest(), digest, "{name}"); + } + } + #[test] fn quote_tracks_report_data_and_rtmr_extensions() { - let mut state = SimulatorState::new(QUOTE_FIXTURE, CCEL_FIXTURE).unwrap(); + let seed = [0x6b; 32]; + let mut state = SimulatorState::new( + Arc::new(TdxGenerator::from_seed(seed).unwrap()), + CCEL_FIXTURE, + None, + ) + .unwrap(); let original_rtmr3 = state.rtmrs[3]; let digest = [0x42; 48]; state.extend_rtmr(3, &digest).unwrap(); @@ -464,17 +649,30 @@ mod tests { let quote = Quote::parse(&state.outblob).unwrap(); let report = quote.report.as_td10().unwrap(); - let fixture = Quote::parse(QUOTE_FIXTURE).unwrap(); - let fixture_report = fixture.report.as_td10().unwrap(); assert_eq!(report.report_data, report_data); assert_eq!(report.mr_config_id, [0u8; 48]); assert_eq!(report.rt_mr3, state.rtmrs[3]); - assert_eq!(report.mr_owner, fixture_report.mr_owner); + assert_eq!(report.mr_owner, [0u8; 48]); + + let host = TdxGenerator::from_seed(seed).unwrap(); + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + dcap_qvl::verify::QuoteVerifier::new(host.root_ca_der()) + .verify(&state.outblob, &host.sample_collateral().unwrap(), now) + .unwrap(); + let wrong = TdxGenerator::from_seed([0x6c; 32]).unwrap(); + assert!(dcap_qvl::verify::QuoteVerifier::new(wrong.root_ca_der()) + .verify(&state.outblob, &host.sample_collateral().unwrap(), now) + .is_err()); } #[test] fn only_rtmr_two_and_three_are_extensible() { - let mut state = SimulatorState::new(QUOTE_FIXTURE, CCEL_FIXTURE).unwrap(); + let mut state = + SimulatorState::new(Arc::new(TdxGenerator::new().unwrap()), CCEL_FIXTURE, None) + .unwrap(); assert!(state.extend_rtmr(0, &[0u8; 48]).is_err()); assert!(state.extend_rtmr(2, &[0u8; 48]).is_ok()); assert!(state.extend_rtmr(3, &[0u8; 48]).is_ok()); @@ -482,6 +680,39 @@ mod tests { assert!(state.extend_rtmr(3, &[0u8; 47]).is_err()); } + #[test] + fn state_updates_are_failure_atomic() { + let mut state = SimulatorState::new( + Arc::new(TdxGenerator::from_seed([0x71; 32]).unwrap()), + CCEL_FIXTURE, + None, + ) + .unwrap(); + let original_quote = state.outblob.clone(); + let original_rtmr3 = state.rtmrs[3]; + + assert!(state.request_quote(&[0x11; 63]).is_err()); + assert_eq!(state.generation, 0); + assert_eq!(state.outblob, original_quote); + + assert!(state.extend_rtmr(3, &[0x22; 47]).is_err()); + assert_eq!(state.rtmrs[3], original_rtmr3); + assert!(state.extend_rtmr(1, &[0x22; 48]).is_err()); + assert_eq!(state.rtmrs[1], replay_boot_rtmrs(CCEL_FIXTURE).unwrap()[1]); + + state.generation = i64::MAX; + assert!(state.request_quote(&[0x33; 64]).is_err()); + assert_eq!(state.generation, i64::MAX); + assert_eq!(state.outblob, original_quote); + state.generation = 0; + + state.request_quote(&[0x33; 64]).unwrap(); + assert_eq!(state.generation, 1); + assert_ne!(state.outblob, original_quote); + state.extend_rtmr(3, &[0x44; 48]).unwrap(); + assert_ne!(state.rtmrs[3], original_rtmr3); + } + #[test] fn boot_rtmrs_replay_the_bundled_ccel() { let rtmrs = replay_boot_rtmrs(CCEL_FIXTURE).unwrap(); diff --git a/dstack/tee-simulator/src/tpm.rs b/dstack/tee-simulator/src/tpm.rs new file mode 100644 index 000000000..31fa318b8 --- /dev/null +++ b/dstack/tee-simulator/src/tpm.rs @@ -0,0 +1,731 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! swtpm-backed GCP vTPM environment, including the pre-provisioned AK +//! template and certificate NV indices consumed by `tpm-attest`. + +use std::{ + io::{Read, Write}, + os::{ + fd::{AsRawFd, FromRawFd}, + unix::fs::{FileTypeExt as _, MetadataExt as _}, + unix::net::UnixStream, + }, + path::Path, + process::{Command, Stdio}, + thread, + time::Duration, +}; + +use anyhow::{bail, Context, Result}; +use aws_nitro_enclaves_nsm_api::api::{Request as NsmRequest, Response as NsmResponse}; +use dstack_types::{AwsPcrReplay, TeeSimulatorConfig}; +use mock_attestation::{nsm::NsmGenerator, parse_seed, server::MockCollateralState}; +use tpm2::{add_command_capability, TpmAlgId, TpmCc, TpmContext}; + +const AK_ECC_CERT: &str = "0x01c10002"; +const AK_ECC_TEMPLATE: &str = "0x01c10003"; +const TPM2_CC_AWS_NSM_REQUEST: u32 = 0x2000_0001; +const VTPM_PROXY_IOC_NEW_DEV: libc::c_ulong = 0xc014_a100; +const VTPM_PROXY_FLAG_TPM2: u32 = 1; + +#[repr(C)] +#[derive(Default)] +struct VtpmProxyNewDev { + flags: u32, + tpm_num: u32, + fd: u32, + major: u32, + minor: u32, +} + +struct NvWriteTemplate { + request: Vec<u8>, +} + +fn read_be_u16(bytes: &[u8], field: &str) -> Result<u16> { + let bytes: [u8; 2] = bytes + .try_into() + .map_err(|_| anyhow::anyhow!("invalid {field} length"))?; + Ok(u16::from_be_bytes(bytes)) +} + +fn read_be_u32(bytes: &[u8], field: &str) -> Result<u32> { + let bytes: [u8; 4] = bytes + .try_into() + .map_err(|_| anyhow::anyhow!("invalid {field} length"))?; + Ok(u32::from_be_bytes(bytes)) +} + +fn command(program: &str, args: &[&str]) -> Result<()> { + let status = Command::new(program) + .args(args) + .env("TPM2TOOLS_TCTI", "device:/dev/tpm0") + .status() + .with_context(|| format!("failed to execute {program}"))?; + if !status.success() { + bail!("{program} failed with status {status}"); + } + Ok(()) +} + +fn device_node_matches(path: &Path, major: u64, minor: u64) -> Result<bool> { + let metadata = match fs_err::metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error.into()), + }; + Ok(metadata.file_type().is_char_device() + && nix::sys::stat::major(metadata.rdev()) == major + && nix::sys::stat::minor(metadata.rdev()) == minor) +} + +fn ensure_device_node(path: &Path, major: &str, minor: &str) -> Result<()> { + let expected_major = major.parse().context("invalid device major")?; + let expected_minor = minor.parse().context("invalid device minor")?; + if device_node_matches(path, expected_major, expected_minor)? { + return Ok(()); + } + match fs_err::symlink_metadata(path) { + Ok(_) => { + // Recheck in case udev published the expected node between the two + // metadata calls above. + if device_node_matches(path, expected_major, expected_minor)? { + return Ok(()); + } + bail!( + "{} exists but is not character device {expected_major}:{expected_minor}", + path.display() + ); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + let path_text = path.to_str().context("device path is not UTF-8")?; + if let Err(error) = command("mknod", &[path_text, "c", major, minor]) { + // udev can publish the node after the existence check but before + // mknod. Accept only the exact character device registered by sysfs. + if device_node_matches(path, expected_major, expected_minor)? { + return Ok(()); + } + return Err(error); + } + Ok(()) +} + +pub fn start_gcp_vtpm(runtime_dir: &Path, config: &TeeSimulatorConfig) -> Result<()> { + if Path::new("/dev/tpmrm0").exists() || Path::new("/dev/tpm0").exists() { + bail!("refusing to start the GCP vTPM simulator when a real TPM is present"); + } + let seed = config + .mock_attestation_seed + .as_deref() + .context("tee_simulator.mock_attestation_seed is required")?; + let base_url = config + .collateral_base_url + .as_deref() + .unwrap_or("http://127.0.0.1:8088"); + let state = MockCollateralState::from_seed(parse_seed(seed)?, base_url)?; + // Keep swtpm state outside /run: swtpm drops privileges to `tss`, and some + // distributions reject its lock file when a parent runtime directory is + // owned by root even if the immediate state directory is writable. + let state_dir = std::env::temp_dir().join(format!("dstack-swtpm-{}", std::process::id())); + fs_err::create_dir_all(&state_dir)?; + let state_arg = format!("dir={}", state_dir.display()); + let pid_file = std::env::temp_dir().join(format!("dstack-swtpm-{}.pid", std::process::id())); + let pid_arg = format!("file={}", pid_file.display()); + command( + "swtpm", + &[ + "chardev", + "--tpm2", + "--tpmstate", + &state_arg, + "--vtpm-proxy", + "--pid", + &pid_arg, + "--daemon", + ], + )?; + fs_err::copy(&pid_file, runtime_dir.join("swtpm.pid"))?; + + for _ in 0..100 { + create_tpm_device_node()?; + if Path::new("/dev/tpmrm0").exists() || Path::new("/dev/tpm0").exists() { + break; + } + thread::sleep(Duration::from_millis(20)); + } + let mut startup_error = None; + for _ in 0..100 { + match command("tpm2_startup", &["-c"]) { + Ok(()) => { + startup_error = None; + break; + } + Err(error) => { + startup_error = Some(error); + thread::sleep(Duration::from_millis(20)); + } + } + } + if let Some(error) = startup_error { + return Err(error).context("GCP vTPM did not become ready"); + } + let replay = config + .gcp_tpm_replay + .as_ref() + .context("tee_simulator.gcp_tpm_replay is required for GCP")?; + validate_gcp_event_log(config, &replay.event_log)?; + replay_gcp_event_log(&replay.event_log)?; + install_gcp_event_log(&replay.event_log)?; + + let template_with_size = state_dir.join("ak.tpm2b-public"); + let generated_public = state_dir.join("ak.public"); + let template = state_dir.join("ak.template"); + let public_pem = state_dir.join("ak.pem"); + let context = state_dir.join("ak.ctx"); + command( + "tpm2_createprimary", + &[ + "-C", + "e", + "-G", + "ecc256:ecdsa-sha256:null", + "-g", + "sha256", + "-a", + "fixedtpm|fixedparent|sensitivedataorigin|userwithauth|restricted|sign|noda", + "-c", + context.to_str().context("invalid AK context path")?, + "-o", + generated_public + .to_str() + .context("invalid generated AK public path")?, + "--template-data", + template_with_size + .to_str() + .context("invalid AK public path")?, + ], + )?; + command( + "tpm2_readpublic", + &[ + "-c", + context.to_str().context("invalid AK context path")?, + "-f", + "pem", + "-o", + public_pem.to_str().context("invalid AK PEM path")?, + ], + )?; + let bytes = fs_err::read(&template_with_size)?; + anyhow::ensure!(!bytes.is_empty(), "invalid TPMT_PUBLIC generated by swtpm"); + fs_err::write(&template, bytes)?; + + let root_cert = state_dir.join("root.pem"); + let root_key = state_dir.join("root-key.pem"); + let ak_cert = state_dir.join("ak-cert.pem"); + let extensions = state_dir.join("ak-cert.ext"); + fs_err::write(&root_cert, state.tpm.root_ca_pem())?; + fs_err::write(&root_key, state.tpm.root_key_pem())?; + fs_err::write( + &extensions, + format!( + "basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\nauthorityInfoAccess=caIssuers;URI:{base_url}/tpm/aia/root.pem\ncrlDistributionPoints=URI:{base_url}/tpm/crl/root.crl\n" + ), + )?; + command( + "openssl", + &[ + "x509", + "-new", + "-force_pubkey", + public_pem.to_str().context("invalid AK PEM path")?, + "-CAkey", + root_key.to_str().context("invalid root key path")?, + "-CA", + root_cert.to_str().context("invalid root cert path")?, + "-CAcreateserial", + "-days", + "30", + "-subj", + "/CN=Mock GCP vTPM AK", + "-extfile", + extensions.to_str().context("invalid extension path")?, + "-outform", + "DER", + "-out", + ak_cert.to_str().context("invalid AK certificate path")?, + ], + )?; + provision_nv(AK_ECC_TEMPLATE, &template)?; + provision_nv(AK_ECC_CERT, &ak_cert)?; + command("tpm2_flushcontext", &["-t"])?; + Ok(()) +} + +fn validate_gcp_event_log(config: &TeeSimulatorConfig, mut bytes: &[u8]) -> Result<()> { + let vm_config: dstack_types::VmConfig = serde_json::from_str( + config + .vm_config + .as_deref() + .context("tee_simulator.vm_config is required for GCP")?, + )?; + let expected = vm_config + .gcp_measurement + .as_ref() + .context("vm_config.gcp_measurement is required for GCP")? + .decode_measurement() + .map_err(anyhow::Error::msg)? + .uki_authenticode_sha256; + let event_log = cc_eventlog::tpm::TpmEventLog::decode(&mut bytes)?; + let actual = event_log + .pcr2_events() + .get(2) + .context("GCP TPM event log is missing the UKI event")? + .digest + .clone(); + anyhow::ensure!( + actual == expected, + "GCP TPM event-log UKI digest does not match measurement.gcp.cbor" + ); + Ok(()) +} + +fn replay_gcp_event_log(mut bytes: &[u8]) -> Result<()> { + let event_log = cc_eventlog::tpm::TpmEventLog::decode(&mut bytes)?; + for event in event_log.events { + let extension = format!("{}:sha256={}", event.pcr_index, hex::encode(event.digest)); + command("tpm2_pcrextend", &[&extension])?; + } + Ok(()) +} + +fn install_gcp_event_log(bytes: &[u8]) -> Result<()> { + let security_root = Path::new("/sys/kernel/security"); + let event_log = security_root.join("tpm0/binary_bios_measurements"); + if event_log.exists() { + anyhow::ensure!( + fs_err::read(&event_log)? == bytes, + "existing simulated TPM event log does not match the image" + ); + return Ok(()); + } + let tpm_dir = event_log.parent().context("TPM event log has no parent")?; + // securityfs does not permit userspace to create a synthetic TPM event + // log hierarchy. Shadow it in this development-only guest before + // publishing the event log that was replayed into the simulated PCRs. + let flags = nix::mount::MsFlags::MS_NOSUID + | nix::mount::MsFlags::MS_NODEV + | nix::mount::MsFlags::MS_NOEXEC; + nix::mount::mount( + Some("dstack-tee-simulator"), + security_root, + Some("tmpfs"), + flags, + Some("mode=0755"), + ) + .context("failed to mount simulated securityfs shadow")?; + fs_err::create_dir_all(tpm_dir) + .context("failed to create TPM event-log directory in securityfs shadow")?; + fs_err::write(event_log, bytes).context("failed to install simulated TPM event log") +} +fn create_tpm_device_node() -> Result<()> { + let sys_dev = Path::new("/sys/class/tpm/tpm0/dev"); + if !sys_dev.exists() { + return Ok(()); + } + let device = fs_err::read_to_string(sys_dev)?; + let (major, minor) = device + .trim() + .split_once(':') + .context("invalid /sys/class/tpm/tpm0/dev")?; + ensure_device_node(Path::new("/dev/tpm0"), major, minor) +} + +fn provision_nv(index: &str, contents: &Path) -> Result<()> { + let size = fs_err::metadata(contents)?.len().to_string(); + command( + "tpm2_nvdefine", + &[index, "-C", "o", "-s", &size, "-a", "ownerread|ownerwrite"], + )?; + command( + "tpm2_nvwrite", + &[ + index, + "-C", + "o", + "-i", + contents.to_str().context("invalid NV input path")?, + ], + ) +} + +/// Run a TPM2 device backed by swtpm while implementing the NitroTPM NSM +/// vendor command at the TPM wire-protocol boundary. All ordinary commands, +/// including EK creation, HMAC sessions, and NV access, are handled by swtpm. +pub fn run_nitro_vtpm(runtime_dir: &Path, config: &TeeSimulatorConfig) -> Result<()> { + if Path::new("/dev/tpmrm0").exists() || Path::new("/dev/tpm0").exists() { + bail!("refusing to start the NitroTPM simulator when a real TPM is present"); + } + let seed = config + .mock_attestation_seed + .as_deref() + .context("tee_simulator.mock_attestation_seed is required")?; + let generator = NsmGenerator::from_seed(parse_seed(seed)?)?; + let (mut simulator, swtpm_stream) = UnixStream::pair()?; + let swtpm_fd = swtpm_stream.as_raw_fd(); + let flags = nix::fcntl::FdFlag::from_bits_truncate( + nix::fcntl::fcntl(swtpm_fd, nix::fcntl::FcntlArg::F_GETFD) + .context("failed to get swtpm socket flags")?, + ); + nix::fcntl::fcntl( + swtpm_fd, + nix::fcntl::FcntlArg::F_SETFD(flags - nix::fcntl::FdFlag::FD_CLOEXEC), + ) + .context("failed to make swtpm socket inheritable")?; + + let state_dir = std::env::temp_dir().join(format!("dstack-nitro-swtpm-{}", std::process::id())); + fs_err::create_dir_all(&state_dir)?; + let state_arg = format!("dir={}", state_dir.display()); + let fd_arg = swtpm_fd.to_string(); + let mut child = Command::new("swtpm") + .args([ + "chardev", + "--tpm2", + "--tpmstate", + &state_arg, + "--fd", + &fd_arg, + "--flags", + "not-need-init,startup-clear", + "--locality", + "allow-set-locality", + ]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .context("failed to start NitroTPM swtpm backend")?; + drop(swtpm_stream); + fs_err::write(runtime_dir.join("swtpm.pid"), child.id().to_string())?; + + let replay = config + .aws_pcr_replay + .as_ref() + .context("tee_simulator.aws_pcr_replay is required for NitroTPM")?; + replay_aws_boot_pcrs(&mut simulator, replay)?; + + let (control, mut proxy, tpm_num) = create_vtpm_proxy()?; + let proxy_thread = thread::spawn(move || { + let _control = control; + let result = proxy_tpm_commands(&mut proxy, &mut simulator, &generator); + if let Err(error) = &result { + tracing::error!(?error, "NitroTPM proxy failed"); + } + result + }); + let sys_dev = format!("/sys/class/tpm/tpm{tpm_num}/dev"); + for _ in 0..100 { + if Path::new(&sys_dev).exists() { + break; + } + thread::sleep(Duration::from_millis(10)); + } + let device = fs_err::read_to_string(&sys_dev).context("vTPM was not registered")?; + let (major, minor) = device + .trim() + .split_once(':') + .context("invalid vTPM device number")?; + ensure_device_node(Path::new("/dev/tpm0"), major, minor)?; + sd_notify::notify(true, &[sd_notify::NotifyState::Ready])?; + let result = proxy_thread + .join() + .map_err(|_| anyhow::anyhow!("NitroTPM proxy thread panicked"))?; + let _ = child.kill(); + result +} + +fn replay_aws_boot_pcrs(backend: &mut UnixStream, replay: &AwsPcrReplay) -> Result<()> { + anyhow::ensure!(replay.version == 1, "unsupported AWS PCR replay version"); + let mut tpm = TpmContext::from_stream( + backend + .try_clone() + .context("failed to clone NitroTPM stream for PCR replay")?, + "NitroTPM replay backend", + ); + for event in &replay.events { + anyhow::ensure!( + matches!(event.pcr, 4 | 7 | 12), + "AWS PCR replay contains unsupported PCR {}", + event.pcr + ); + anyhow::ensure!( + event.digest.len() == 48, + "AWS PCR replay event digest must be SHA-384" + ); + tpm.pcr_extend(event.pcr.into(), &event.digest, TpmAlgId::Sha384) + .with_context(|| { + format!( + "failed to replay {} into PCR{}", + event.event_type, event.pcr + ) + })?; + } + for (index, expected) in [(4u16, &replay.pcr4), (7, &replay.pcr7), (12, &replay.pcr12)] { + anyhow::ensure!(expected.len() == 48, "expected PCR{index} must be SHA-384"); + let actual = tpm.pcr_read_single(index.into(), TpmAlgId::Sha384)?; + anyhow::ensure!( + actual == *expected, + "replayed PCR{index} mismatch: expected={}, actual={}", + hex::encode(expected), + hex::encode(actual) + ); + } + Ok(()) +} + +fn create_vtpm_proxy() -> Result<(std::fs::File, std::fs::File, u32)> { + let control = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open("/dev/vtpmx") + .context("failed to open /dev/vtpmx; load tpm_vtpm_proxy")?; + let mut new_dev = VtpmProxyNewDev { + flags: VTPM_PROXY_FLAG_TPM2, + ..Default::default() + }; + let rc = unsafe { libc::ioctl(control.as_raw_fd(), VTPM_PROXY_IOC_NEW_DEV, &mut new_dev) }; + anyhow::ensure!( + rc == 0, + "failed to create vTPM proxy device: {}", + std::io::Error::last_os_error() + ); + // SAFETY: the successful ioctl returned ownership of this new descriptor. + Ok(( + control, + unsafe { std::fs::File::from_raw_fd(new_dev.fd as i32) }, + new_dev.tpm_num, + )) +} + +fn proxy_tpm_commands( + proxy: &mut std::fs::File, + backend: &mut UnixStream, + generator: &NsmGenerator, +) -> Result<()> { + let mut nv_write = None; + let mut nsm_response = None; + loop { + let mut command = vec![0u8; 65_536]; + let size = loop { + match proxy.read(&mut command) { + Ok(size) => break size, + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(error) if error.raw_os_error() == Some(libc::EPIPE) => { + thread::sleep(Duration::from_millis(10)); + } + Err(error) => return Err(error).context("failed to read vTPM command"), + } + }; + if size == 0 { + return Ok(()); + } + command.truncate(size); + anyhow::ensure!(command.len() >= 10, "truncated TPM command"); + let code = read_be_u32(&command[6..10], "TPM command code")?; + let response = if code == TPM2_CC_AWS_NSM_REQUEST { + let template = nv_write + .as_ref() + .context("NitroTPM vendor command without an NV request")?; + nsm_response = Some(handle_nsm_vendor_command(generator, template, backend)?); + tpm_success_response() + } else if code == TpmCc::NvRead.to_u32() && nsm_response.is_some() { + nv_read_response( + &command, + nsm_response.as_deref().context("missing NSM response")?, + )? + } else if code == TpmCc::NvReadPublic.to_u32() && nsm_response.is_some() { + let mut response = transact(backend, &command)?; + set_nv_public_size( + &mut response, + nsm_response + .as_deref() + .context("missing NSM response")? + .len(), + )?; + response + } else { + // NitroTPM permits an 8 KiB message NV space while libtpms caps a + // single NV index at 2 KiB. Define the backing index at that limit; + // the proxy virtualizes its public size and reads after the vendor + // command, while swtpm still handles its lifecycle and auth setup. + if code == TpmCc::NvDefineSpace.to_u32() && command.ends_with(&8192u16.to_be_bytes()) { + let end = command.len(); + command[end - 2..].copy_from_slice(&2048u16.to_be_bytes()); + } + if code == TpmCc::NvWrite.to_u32() { + if let Some(template) = parse_nv_write(&command)? { + nv_write = Some(template); + } + } + let mut response = transact(backend, &command)?; + advertise_nsm_vendor_command(code, &mut response)?; + response + }; + proxy + .write_all(&response) + .context("failed to write vTPM response")?; + } +} + +fn advertise_nsm_vendor_command(code: u32, response: &mut Vec<u8>) -> Result<()> { + if code == TpmCc::GetCapability.to_u32() { + *response = add_command_capability(response, TPM2_CC_AWS_NSM_REQUEST)?; + } + Ok(()) +} + +fn parse_nv_write(command: &[u8]) -> Result<Option<NvWriteTemplate>> { + // sessions header + auth handle + NV index + authorizationSize + if command.len() < 24 { + return Ok(None); + } + let index = read_be_u32(&command[14..18], "NV index")?; + if !(0x0100_0000..=0x01ff_ffff).contains(&index) { + return Ok(None); + } + let auth_size = read_be_u32(&command[18..22], "NV authorization size")? as usize; + let data_size_pos = 22usize + .checked_add(auth_size) + .context("NV write size overflow")?; + anyhow::ensure!(command.len() >= data_size_pos + 4, "truncated NV write"); + let data_size = read_be_u16( + &command[data_size_pos..data_size_pos + 2], + "NV write data size", + )? as usize; + let data_start = data_size_pos + 2; + let data_end = data_start + data_size; + anyhow::ensure!(command.len() >= data_end + 2, "truncated NV write data"); + let request = command[data_start..data_end].to_vec(); + Ok(Some(NvWriteTemplate { request })) +} + +fn handle_nsm_vendor_command( + generator: &NsmGenerator, + template: &NvWriteTemplate, + backend: &mut UnixStream, +) -> Result<Vec<u8>> { + let request: NsmRequest = serde_cbor::from_slice(&template.request)?; + let response = match request { + NsmRequest::Attestation { user_data, .. } => { + let mut tpm = TpmContext::from_stream( + backend + .try_clone() + .context("failed to clone NitroTPM stream")?, + "NitroTPM backend", + ); + let pcrs = [4u16, 7, 8, 12, 14] + .into_iter() + .map(|index| Ok((index, tpm.pcr_read_single(index.into(), TpmAlgId::Sha384)?))) + .collect::<Result<_>>()?; + let document = generator.attest_with_pcrs( + user_data.as_ref().map(|v| v.as_slice()).unwrap_or_default(), + pcrs, + )?; + NsmResponse::Attestation { document } + } + _ => anyhow::bail!("unsupported NitroTPM NSM request"), + }; + Ok(serde_cbor::to_vec(&response)?) +} + +fn set_nv_public_size(response: &mut [u8], size: usize) -> Result<()> { + anyhow::ensure!(response.len() >= 26, "truncated NV_ReadPublic response"); + let policy_size_pos = 22; + let policy_size = read_be_u16( + &response[policy_size_pos..policy_size_pos + 2], + "NV policy size", + )? as usize; + let data_size_pos = policy_size_pos + 2 + policy_size; + anyhow::ensure!( + response.len() >= data_size_pos + 2, + "truncated NV public area" + ); + response[data_size_pos..data_size_pos + 2].copy_from_slice(&(size as u16).to_be_bytes()); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gcp_event_log_is_bound_to_vm_measurement() { + let fixture = include_bytes!("../../cc-eventlog/samples/tpm_eventlog.bin"); + let fixture_hash = + hex::decode("9ab14a46f858662a89adc102d2a57a13f52f75c1769d65a4c34edbbfc8855f0f") + .unwrap(); + let image_hash = vec![0x5a; 32]; + let offset = fixture + .windows(fixture_hash.len()) + .position(|window| window == fixture_hash) + .unwrap(); + let mut event_log = fixture.to_vec(); + event_log[offset..offset + image_hash.len()].copy_from_slice(&image_hash); + + let measurement = dstack_types::GcpOsImageMeasurement::new(image_hash).unwrap(); + let document = + dstack_types::GcpOsImageMeasurementDocument::from_measurement(Vec::new(), measurement); + let mut config = TeeSimulatorConfig { + vm_config: Some(serde_json::json!({ "gcp_measurement": document }).to_string()), + ..Default::default() + }; + validate_gcp_event_log(&config, &event_log).unwrap(); + + config.vm_config = Some("{}".into()); + assert!(validate_gcp_event_log(&config, &event_log).is_err()); + } +} + +fn nv_read_response(command: &[u8], contents: &[u8]) -> Result<Vec<u8>> { + anyhow::ensure!(command.len() >= 4, "truncated NV_Read command"); + let size = read_be_u16( + &command[command.len() - 4..command.len() - 2], + "NV read size", + )? as usize; + let offset = read_be_u16(&command[command.len() - 2..], "NV read offset")? as usize; + let end = (offset + size).min(contents.len()); + anyhow::ensure!(offset <= end, "invalid NV_Read offset"); + let data = &contents[offset..end]; + let parameter_size = 2 + data.len(); + let total_size = 10 + 4 + parameter_size + 5; + let mut response = Vec::with_capacity(total_size); + response.extend_from_slice(&0x8002u16.to_be_bytes()); + response.extend_from_slice(&(total_size as u32).to_be_bytes()); + response.extend_from_slice(&0u32.to_be_bytes()); + response.extend_from_slice(&(parameter_size as u32).to_be_bytes()); + response.extend_from_slice(&(data.len() as u16).to_be_bytes()); + response.extend_from_slice(data); + response.extend_from_slice(&[0, 0, 0, 0, 0]); + Ok(response) +} + +fn transact(stream: &mut UnixStream, command: &[u8]) -> Result<Vec<u8>> { + stream.write_all(command)?; + let mut header = [0u8; 10]; + stream.read_exact(&mut header)?; + let size = read_be_u32(&header[2..6], "TPM response size")? as usize; + anyhow::ensure!(size >= header.len(), "invalid TPM response size"); + let mut response = Vec::with_capacity(size); + response.extend_from_slice(&header); + response.resize(size, 0); + stream.read_exact(&mut response[10..])?; + Ok(response) +} + +fn tpm_success_response() -> Vec<u8> { + [0x80, 0x01, 0, 0, 0, 10, 0, 0, 0, 0].to_vec() +} diff --git a/dstack/tee-simulator/tests/process_e2e.rs b/dstack/tee-simulator/tests/process_e2e.rs new file mode 100644 index 000000000..df13689eb --- /dev/null +++ b/dstack/tee-simulator/tests/process_e2e.rs @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + process::{Child, Command, Stdio}, + time::{Duration, Instant}, +}; + +use mock_attestation::server::MockCollateralState; + +/// The simulator derives a development PKI before it mounts anything, so the +/// budget has to cover process spawn plus four key generations, not just the +/// mount. A debug build on a loaded CI runner is an order of magnitude slower +/// than a local release-ish one, so keep the headroom generous: the test still +/// returns as soon as the mountpoint appears, and a dead child is reported +/// immediately rather than waited out. +const READY_TIMEOUT: Duration = Duration::from_secs(30); +const POLL_INTERVAL: Duration = Duration::from_millis(50); + +struct ChildGuard(Child); +impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +async fn start(platform: &str, seed: [u8; 32]) -> (tempfile::TempDir, ChildGuard) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("tee-simulator.json"); + fs_err::write( + &path, + serde_json::json!({ + "vm_config": "{}", + "mr_config": "{\"version\":3,\"app_id\":\"\",\"compose_hash\":\"\",\"key_provider\":\"none\"}", + "platform": platform, + "mock_attestation_seed": hex::encode(seed), + "collateral_base_url": "http://127.0.0.1:18088" + }) + .to_string(), + ) + .unwrap(); + let mountpoint = dir.path().join("tsm"); + let mut args = vec![ + "--config".to_string(), + path.display().to_string(), + "--runtime-dir".to_string(), + dir.path().display().to_string(), + "--dmi-root".to_string(), + dir.path().join("dmi").display().to_string(), + ]; + if matches!(platform, "dstack-tdx" | "dstack-amd-sev-snp") { + args.extend(["--mountpoint".into(), mountpoint.display().to_string()]); + } + let mut child = ChildGuard( + Command::new(env!("CARGO_BIN_EXE_dstack-tee-simulator")) + .args(args) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(), + ); + let deadline = Instant::now() + READY_TIMEOUT; + loop { + let ready = match platform { + "dstack-tdx" => mountpoint.join("com.intel.dcap/outblob").exists(), + "dstack-amd-sev-snp" => mountpoint.join("provider").exists(), + _ => false, + }; + if ready { + return (dir, child); + } + // A simulator that already exited never becomes ready. Say which one + // and with what status instead of reporting a timeout that hides it. + if let Some(status) = child.0.try_wait().unwrap() { + panic!("{platform} simulator exited before becoming ready: {status}"); + } + assert!( + Instant::now() < deadline, + "{platform} simulator did not become ready within {READY_TIMEOUT:?}" + ); + tokio::time::sleep(POLL_INTERVAL).await; + } +} + +#[tokio::test] +async fn separate_simulator_process_imports_config_seed_for_tsm_platforms() { + let seed = [0x71; 32]; + let host = MockCollateralState::from_seed(seed, "http://127.0.0.1:18088").unwrap(); + + let (dir, child) = start("dstack-tdx", seed).await; + let rd = [0x21; 64]; + let provider = dir.path().join("tsm/com.intel.dcap"); + fs_err::write(provider.join("inblob"), rd).unwrap(); + let quote = fs_err::read(provider.join("outblob")).unwrap(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + dcap_qvl::verify::QuoteVerifier::new(host.tdx.root_ca_der()) + .verify("e, &host.tdx.sample_collateral().unwrap(), now) + .unwrap(); + + // The guest verifier's trust anchor comes from the simulator, never from + // the host, so the published root must be the one that signed this quote. + let published = dstack_attest::trust_anchors::load_anchors(&dir.path().join("attestation")) + .unwrap() + .expect("simulator should publish external trust anchors"); + let root = fs_err::read(published.tdx.unwrap()).unwrap(); + assert_eq!(String::from_utf8(root).unwrap(), host.tdx.root_ca_pem()); + drop(child); + + let (dir, child) = start("dstack-amd-sev-snp", seed).await; + let provider = dir.path().join("tsm"); + let entry = provider.join(format!("dstack-{}", std::process::id())); + use std::io::Write as _; + std::fs::OpenOptions::new() + .write(true) + .open(entry.join("inblob")) + .unwrap() + .write_all(&rd) + .unwrap(); + let report = fs_err::read(entry.join("outblob")).unwrap(); + let cert_chain = fs_err::read(entry.join("certs")).unwrap(); + sev_snp_qvl::QuoteVerifier::new_with_root( + sev_snp_qvl::AmdSnpProduct::Milan, + host.sev_snp.root_ca_pem().into_bytes(), + ) + .verify(&report, &[cert_chain], &rd) + .unwrap(); + drop(child); +} diff --git a/dstack/test-scripts/snp-e2e-smoke.sh b/dstack/test-scripts/snp-e2e-smoke.sh deleted file mode 100755 index ea7d1373e..000000000 --- a/dstack/test-scripts/snp-e2e-smoke.sh +++ /dev/null @@ -1,553 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> -# -# SPDX-License-Identifier: Apache-2.0 -# -# Manual AMD SEV-SNP hardware smoke for dstack-managed KMS/app key release. -# -# This is intentionally not a CI script. It requires an SNP-capable host with the -# AMDSEV QEMU/OVMF build used by the PR smoke, sudo for QEMU/KVM, and locally -# built release binaries. -# -# Minimal setup used by the original smoke: -# cargo build --manifest-path dstack/Cargo.toml --release -p dstack-vmm -p supervisor -p dstack-kms -# export DSTACK_SNP_SMOKE_BIN_DIR=$PWD/dstack/target/release -# export DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB=1 # lab hosts only; auth API policy -# test-scripts/snp-e2e-smoke.sh -# -# Useful overrides: -# DSTACK_SNP_SMOKE_BASE=$HOME/dstack-snp-e2e -# DSTACK_SNP_SMOKE_REPO=$PWD/dstack -# DSTACK_SNP_SMOKE_QEMU=/opt/AMDSEV/usr/local/bin/qemu-system-x86_64 -# DSTACK_SNP_SMOKE_OVMF=/opt/AMDSEV/usr/local/share/qemu/OVMF.fd -# DSTACK_SNP_SMOKE_IMAGE_URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.5.11/dstack-dev-0.5.11.tar.gz -# DSTACK_SNP_SMOKE_IMAGE_NAME=dstack-dev-0.5.11-snp-dnsfix -# DSTACK_SNP_SMOKE_ALLOW_OLD_QEMU=1 # bypasses the QEMU >= 10 preflight -# -# Host/image caveat: QEMU >= 10 is necessary but not sufficient. One local SNP -# host could boot a newer Lit SNP guest kernel but reset before Linux serial -# output with the stock meta-dstack v0.5.11 6.9.0-dstack kernel. If this smoke -# stops after `EFI stub: Loaded initrd ...` with `cpus are not resettable`, first -# validate the guest image/kernel on that host before debugging KMS or apps. -# -# Guest userspace caveat: rebuilding the host-side PR binaries is not enough for -# full app-key success if the downloaded meta-dstack image still embeds an older -# dstack-util/dstack-attest. On that skewed image the app guest can reach -# dstack-prepare.sh and fail at GetTempCaCert/GetAppKey with: -# amd sev-snp cert_chain must contain either ASK and VCEK certificates or one -# kernel certificate table auxblob -# For full SNP_APP_CONTAINER_STARTED / GetAppKey success, use a coherent -# meta-dstack guest image that includes the same PR cert-chain/KDS fallback code. -# If AMD KDS throttles VCEK/cert-chain retrieval (for example HTTP 429 from -# kdsintf.amd.com), keep verification fail-closed and set -# DSTACK_SNP_SMOKE_KDS_BASE_URL to a trusted AMD-KDS-compatible mirror/cache -# base, e.g. https://mirror.example.com/vcek/v1. For a path-prefix relay, set -# the full relayed base, e.g.: -# https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1 -# This is an external collateral-fetch boundary, not a guest boot or KMS startup -# failure. -# One reproducible way is to build os/yocto from the same monorepo revision, -# set the Yocto build MACHINE to `sev-snp` (not the -# default `tdx`, otherwise the guest kernel can miss AMD memory-encryption -# support and reset immediately after OVMF loads the kernel/initrd), then point -# DSTACK_SNP_SMOKE_IMAGE_NAME at the resulting dstack-dev image directory. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CORE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -BASE="${DSTACK_SNP_SMOKE_BASE:-$HOME/dstack-snp-e2e}" -REPO="${DSTACK_SNP_SMOKE_REPO:-$CORE_DIR}" -BIN="${DSTACK_SNP_SMOKE_BIN_DIR:-$REPO/target/release}" -ART="$BASE/artifacts" -LOG="$ART/snp-e2e-smoke.log" -IMAGE_NAME="${DSTACK_SNP_SMOKE_IMAGE_NAME:-dstack-dev-0.5.11-snp-dnsfix}" -IMAGE_URL="${DSTACK_SNP_SMOKE_IMAGE_URL:-https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.5.11/dstack-dev-0.5.11.tar.gz}" -QEMU_PATH="${DSTACK_SNP_SMOKE_QEMU:-/opt/AMDSEV/usr/local/bin/qemu-system-x86_64}" -OVMF_PATH="${DSTACK_SNP_SMOKE_OVMF:-/opt/AMDSEV/usr/local/share/qemu/OVMF.fd}" -HOST_ART_PORT="${DSTACK_SNP_SMOKE_HOST_ART_PORT:-18080}" -AUTH_PORT="${DSTACK_SNP_SMOKE_AUTH_PORT:-18081}" -KMS_HOST_PORT="${DSTACK_SNP_SMOKE_KMS_HOST_PORT:-15443}" -STRICT_KMS_HOST_PORT="${DSTACK_SNP_SMOKE_STRICT_KMS_HOST_PORT:-15444}" -APP_HOST_PORT="${DSTACK_SNP_SMOKE_APP_HOST_PORT:-15543}" -STRICT_APP_HOST_PORT="${DSTACK_SNP_SMOKE_STRICT_APP_HOST_PORT:-15544}" -VMM_PORT="${DSTACK_SNP_SMOKE_VMM_PORT:-18082}" -VMM_URL="${DSTACK_SNP_SMOKE_VMM_URL:-http://127.0.0.1:$VMM_PORT}" -HOST_API_PORT="${DSTACK_SNP_SMOKE_HOST_API_PORT:-11100}" -ALLOW_OUT_OF_DATE_TCB="${DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB:-0}" -RUN_STRICT_TCB_PROBE="${DSTACK_SNP_SMOKE_STRICT_TCB_PROBE:-1}" -ALLOW_OLD_QEMU="${DSTACK_SNP_SMOKE_ALLOW_OLD_QEMU:-0}" - -need() { - if ! command -v "$1" >/dev/null 2>&1; then - echo "missing required command: $1" >&2 - exit 1 - fi -} - -need curl -need jq -need openssl -need python3 -need sudo - -test -x "$BIN/dstack-vmm" || { echo "missing $BIN/dstack-vmm; run cargo build --release -p dstack-vmm" >&2; exit 1; } -test -x "$BIN/supervisor" || { echo "missing $BIN/supervisor; run cargo build --release -p supervisor" >&2; exit 1; } -test -x "$BIN/dstack-kms" || { echo "missing $BIN/dstack-kms; run cargo build --release -p dstack-kms" >&2; exit 1; } -test -x "$QEMU_PATH" || { echo "missing SNP QEMU: $QEMU_PATH" >&2; exit 1; } -test -r "$OVMF_PATH" || { echo "missing SNP OVMF: $OVMF_PATH" >&2; exit 1; } -test -f "$REPO/vmm/src/vmm-cli.py" || { echo "missing vmm-cli.py; set DSTACK_SNP_SMOKE_REPO" >&2; exit 1; } - -qemu_version_output=$("$QEMU_PATH" --version | head -1) -qemu_version=$(printf '%s\n' "$qemu_version_output" | sed -n 's/.*version \([0-9][0-9]*\)\.\([0-9][0-9]*\).*/\1.\2/p') -qemu_major=${qemu_version%%.*} -if [[ -z "$qemu_version" ]]; then - echo "Warning: could not parse QEMU version from: $qemu_version_output" >&2 -elif (( qemu_major < 10 )) && [[ "$ALLOW_OLD_QEMU" != "1" ]]; then - cat >&2 <<EOF -Unsupported SNP smoke QEMU version: $qemu_version_output - -The known-good PR #703 smoke used AMDSEV QEMU 10.0.2. A local Chipotle -attempt with QEMU 9.1.0 reached OVMF/EFI stub and then exited with: - qemu-system-x86_64: cpus are not resettable, terminating - -Use an AMDSEV QEMU >= 10 build, or set DSTACK_SNP_SMOKE_ALLOW_OLD_QEMU=1 -if you intentionally want to reproduce/debug the older-QEMU failure. -EOF - exit 1 -fi - -mkdir -p "$ART" "$BASE/images" "$BASE/run" "$BASE/http-root" -exec > >(tee "$LOG") 2>&1 - -echo "== SNP E2E smoke start: $(date -Is) ==" -echo "repo=$REPO" -echo "repo_head=$(git -C "$REPO" rev-parse --short=16 HEAD 2>/dev/null || echo unknown)" -echo "qemu=$QEMU_PATH" -echo "qemu_version=$qemu_version_output" -echo "ovmf_sha256=$(sha256sum "$OVMF_PATH" | awk '{print $1}')" -echo "image=$IMAGE_NAME" -if [[ -n "${DSTACK_SNP_SMOKE_KDS_BASE_URL:-}" ]]; then - echo "amd_kds_base_url=${DSTACK_SNP_SMOKE_KDS_BASE_URL}" -fi - -cleanup() { - set +e - if [[ -f "$BASE/vmm.pid" ]]; then sudo kill "$(cat "$BASE/vmm.pid")" 2>/dev/null || true; fi - if [[ -f "$BASE/artifacts-http.pid" ]]; then kill "$(cat "$BASE/artifacts-http.pid")" 2>/dev/null || true; fi - if [[ -f "$BASE/auth.pid" ]]; then kill "$(cat "$BASE/auth.pid")" 2>/dev/null || true; fi - sudo pkill -f "$BIN/dstack-vmm" 2>/dev/null || true - sudo pkill -f "qemu-system-x86_64.*$BASE" 2>/dev/null || true - sudo pkill -f "$BASE/images" 2>/dev/null || true - if command -v fuser >/dev/null 2>&1; then - fuser -k "${HOST_ART_PORT}/tcp" "${AUTH_PORT}/tcp" "${KMS_HOST_PORT}/tcp" "${STRICT_KMS_HOST_PORT}/tcp" "${APP_HOST_PORT}/tcp" "${STRICT_APP_HOST_PORT}/tcp" "${VMM_PORT}/tcp" 2>/dev/null || true - fi -} -trap cleanup EXIT -cleanup -sudo pkill -f "$BIN/supervisor" 2>/dev/null || true -sudo rm -rf "$BASE/run"/* "$BASE/tmp"/* - -cp "$BIN/dstack-kms" "$BASE/http-root/dstack-kms" -chmod +x "$BASE/http-root/dstack-kms" - -if [[ ! -d "$BASE/images/$IMAGE_NAME" ]]; then - echo "== Downloading/extracting $IMAGE_NAME ==" - curl -L "$IMAGE_URL" -o "$BASE/$IMAGE_NAME.tar.gz" - mkdir -p "$BASE/images/$IMAGE_NAME" - tar -xzf "$BASE/$IMAGE_NAME.tar.gz" -C "$BASE/images/$IMAGE_NAME" --strip-components=1 -fi -cp "$OVMF_PATH" "$BASE/images/$IMAGE_NAME/ovmf.fd" -tmp_metadata="$(mktemp)" -jq '.bios = "ovmf.fd"' "$BASE/images/$IMAGE_NAME/metadata.json" >"$tmp_metadata" -mv "$tmp_metadata" "$BASE/images/$IMAGE_NAME/metadata.json" -jq . "$BASE/images/$IMAGE_NAME/metadata.json" | tee "$ART/image-metadata.json" - -cat >"$BASE/auth-server.py" <<'PY' -from http.server import BaseHTTPRequestHandler, HTTPServer -import json -import os -import time - - -ALLOW_OUT_OF_DATE_TCB = os.environ.get("ALLOW_OUT_OF_DATE_TCB") == "1" - - -class H(BaseHTTPRequestHandler): - def _send(self, obj, status=200): - body = json.dumps(obj).encode() - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, fmt, *args): - print(time.strftime("%Y-%m-%dT%H:%M:%S"), self.path, fmt % args, flush=True) - - def do_GET(self): - self._send({ - "status": "ok", - "kmsContractAddr": "0x0000000000000000000000000000000000000000", - "ethRpcUrl": "", - "gatewayAppId": "", - "chainId": 1, - "appImplementation": "0x0000000000000000000000000000000000000000", - }) - - def do_POST(self): - length = int(self.headers.get("Content-Length", "0") or 0) - body = self.rfile.read(length) - try: - data = json.loads(body or b"{}") - except Exception: - data = {} - summary = {k: data.get(k) for k in ["attestationMode", "tcbStatus", "advisoryIds"] if k in data} - for key in ["appId", "mrAggregated", "osImageHash", "composeHash", "instanceId"]: - if key in data: - summary[key] = str(data[key])[:96] - print(json.dumps({"path": self.path, "summary": summary}), flush=True) - - # TCB/advisory policy belongs to the auth API. The normal smoke can - # explicitly allow an OutOfDate lab host; the /strict auth namespace is - # used by the negative probe to prove denial comes from auth policy, not - # KMS-local config. - if self.path.endswith("/bootAuth/app"): - tcb_status = data.get("tcbStatus") or "" - advisory_ids = data.get("advisoryIds") or [] - strict_tcb = self.path.startswith("/strict/") - if tcb_status not in ("", "UpToDate"): - if strict_tcb or tcb_status != "OutOfDate" or not ALLOW_OUT_OF_DATE_TCB: - self._send({ - "isAllowed": False, - "gatewayAppId": "", - "reason": f"tcb_status is not allowed by auth api: {tcb_status}", - }) - return - if advisory_ids: - self._send({ - "isAllowed": False, - "gatewayAppId": "", - "reason": f"advisory_id is not allowed by auth api: {advisory_ids[0]}", - }) - return - - self._send({"isAllowed": True, "gatewayAppId": "", "reason": "snp smoke auth"}) - - -HTTPServer(("0.0.0.0", int(os.environ["AUTH_PORT"])), H).serve_forever() -PY - -(cd "$BASE/http-root" && python3 -m http.server "$HOST_ART_PORT" >"$ART/artifacts-http.log" 2>&1 & echo $! >"$BASE/artifacts-http.pid") -AUTH_PORT="$AUTH_PORT" ALLOW_OUT_OF_DATE_TCB="$ALLOW_OUT_OF_DATE_TCB" python3 "$BASE/auth-server.py" >"$ART/auth-server.log" 2>&1 & echo $! >"$BASE/auth.pid" -sleep 1 -curl -fsS "http://127.0.0.1:$HOST_ART_PORT/dstack-kms" -o /dev/null -curl -fsS "http://127.0.0.1:$AUTH_PORT/" | jq . | tee "$ART/auth-info.json" - -cat >"$BASE/vmm.toml" <<EOF -workers = 8 -max_blocking = 64 -ident = "dstack SNP smoke VMM" -temp_dir = "$BASE/tmp" -keep_alive = 10 -log_level = "debug" -address = "127.0.0.1" -port = $VMM_PORT -reuse = true -kms_url = "https://127.0.0.1:$KMS_HOST_PORT" -event_buffer_size = 50 -node_name = "snp-smoke" -run_path = "$BASE/run" - -[image] -path = "$BASE/images" -registry = "" - -[cvm] -platform = "amd-sev-snp" -qemu_path = "$QEMU_PATH" -kms_urls = ["https://10.0.2.2:$KMS_HOST_PORT"] -gateway_urls = [] -pccs_url = "" -docker_registry = "" -cid_start = 1400 -cid_pool_size = 200 -max_allocable_vcpu = 16 -max_allocable_memory_in_mb = 65536 -qmp_socket = false -user = "" -use_mrconfigid = true -qemu_pci_hole64_size = 0 -qemu_hotplug_off = false -host_share_mode = "vhd" -qgs_port = 4050 - -[cvm.product] -sys_vendor = "dstack" -product_name = "dstack" - -[cvm.networking] -mode = "user" -net = "10.0.2.0/24" -dhcp_start = "10.0.2.10" -restrict = false -forward_service_enabled = false - -[cvm.port_mapping] -enabled = true -address = "127.0.0.1" -range = [{ protocol = "tcp", from = 1, to = 20000 }] - -[cvm.auto_restart] -enabled = false -interval = 20 - -[cvm.gpu] -enabled = false -listing = [] -exclude = [] -include = [] -allow_attach_all = false - -[gateway] -base_domain = "localhost" -port = 8082 -agent_port = 8090 - -[auth] -enabled = false -tokens = [] - -[supervisor] -exe = "$BIN/supervisor" -sock = "$BASE/run/supervisor.sock" -pid_file = "$BASE/run/supervisor.pid" -log_file = "$ART/supervisor.log" -detached = false -auto_start = true - -[host_api] -ident = "dstack SNP smoke VMM" -address = "vsock:0xffffffff" -port = $HOST_API_PORT - -[key_provider] -enabled = true -address = "127.0.0.1" -port = 3443 -EOF - -# Redirect to a user-owned artifact file; only the VMM process itself needs sudo. -# shellcheck disable=SC2024 -sudo "$BIN/dstack-vmm" -c "$BASE/vmm.toml" serve >"$ART/vmm.log" 2>&1 & echo $! >"$BASE/vmm.pid" -for i in $(seq 1 60); do - if python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" lsvm --json >/dev/null 2>&1; then break; fi - sleep 1 - if [[ $i -eq 60 ]]; then echo "VMM did not become ready"; tail -80 "$ART/vmm.log"; exit 1; fi -done -echo "== VMM ready ==" - -write_kms_config() { - local auth_prefix="$1" - local auth_url="http://10.0.2.2:$AUTH_PORT" - if [[ -n "$auth_prefix" ]]; then - auth_url="$auth_url/$auth_prefix" - fi - cat >"$BASE/http-root/kms.toml" <<EOF -[rpc] -address = "0.0.0.0" -port = 8000 - -[rpc.tls] -key = "/dstack/kms-certs/rpc.key" -certs = "/dstack/kms-certs/rpc.crt" - -[rpc.tls.mutual] -ca_certs = "/dstack/kms-certs/tmp-ca.crt" -mandatory = false - -[core] -cert_dir = "/dstack/kms-certs" -admin_token_hash = "00" -pccs_url = "" -enforce_self_authorization = true -amd_kds_base_url = "${DSTACK_SNP_SMOKE_KDS_BASE_URL:-}" -sev_snp_key_release = true - -[core.metrics] -enabled = true - -[core.image] -verify = false -cache_dir = "/dstack/kms-images" -download_url = "" -download_timeout = "2m" - -[core.auth_api] -type = "webhook" - -[core.auth_api.webhook] -url = "$auth_url" - -[core.onboard] -enabled = true -auto_bootstrap_domain = "10.0.2.2" -EOF -} - -DNS_INIT_SCRIPT=$(cat <<'SH' -set -eux -mkdir -p /etc/docker -cat >/etc/docker/daemon.json <<'JSON' -{"dns":["10.0.2.3","1.1.1.1","8.8.8.8"]} -JSON -rm -f /etc/resolv.conf -printf 'nameserver 10.0.2.3\nnameserver 1.1.1.1\nnameserver 8.8.8.8\noptions timeout:2 attempts:3\n' >/etc/resolv.conf -if command -v systemctl >/dev/null 2>&1 && systemctl is-active docker >/dev/null 2>&1; then - systemctl restart docker -fi -SH -) - -KMS_BASH_SCRIPT=$(cat <<'SH' -set -eux -mkdir -p /dstack/kms-certs /dstack/kms-images -curl -fsS http://10.0.2.2:__DSTACK_HOST_ART_PORT__/dstack-kms -o /dstack/dstack-kms -curl -fsS http://10.0.2.2:__DSTACK_HOST_ART_PORT__/kms.toml -o /dstack/kms.toml -chmod +x /dstack/dstack-kms -echo SNP_KMS_CONTAINER_STARTED -RUST_LOG=info /dstack/dstack-kms -c /dstack/kms.toml -SH -) -KMS_BASH_SCRIPT=${KMS_BASH_SCRIPT/__DSTACK_HOST_ART_PORT__/$HOST_ART_PORT} -KMS_BASH_SCRIPT=${KMS_BASH_SCRIPT//__DSTACK_HOST_ART_PORT__/$HOST_ART_PORT} - -deploy_kms() { - local name="$1" - local auth_prefix="$2" - local host_port="$3" - write_kms_config "$auth_prefix" - cat >"$BASE/kms-compose.yaml" <<'YAML' -services: - kms: - image: debian:bookworm-slim - command: sh -c 'echo unused-container-compose; sleep 300' -YAML - python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" compose --docker-compose "$BASE/kms-compose.yaml" --name "$name" --public-logs --public-sysinfo --no-instance-id --output "$BASE/$name.app-compose.json" | tee "$ART/$name-compose-create.txt" >&2 - jq --arg init_script "$DNS_INIT_SCRIPT" --arg bash_script "$KMS_BASH_SCRIPT" '.storage_fs="ext4" | .init_script=$init_script | .runner="bash" | .bash_script=$bash_script | del(.docker_compose_file)' "$BASE/$name.app-compose.json" >"$BASE/$name.app-compose.json.tmp" - mv "$BASE/$name.app-compose.json.tmp" "$BASE/$name.app-compose.json" - python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" deploy --name "$name" --compose "$BASE/$name.app-compose.json" --image "$IMAGE_NAME" --port "tcp:127.0.0.1:$host_port:8000" --vcpu 2 --memory 4096 --disk 20G | tee "$ART/$name-deploy.txt" >&2 - sed -n 's/Created VM with ID: //p' "$ART/$name-deploy.txt" | tail -1 -} - -wait_for_kms_metrics() { - local vm_id="$1" - local host_port="$2" - local label="$3" - for i in $(seq 1 240); do - if curl -kfsS "https://127.0.0.1:$host_port/metrics" >/dev/null 2>&1; then echo "$label KMS runtime ready after ${i}s"; break; fi - sleep 2 - if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for $label KMS..."; python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$vm_id" -n 30 || true; fi - if [[ $i -eq 240 ]]; then echo "$label KMS did not become ready"; python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$vm_id" -n 200 || true; exit 1; fi - done -} - -kms_key_provider_id() { - local kms_port="$1" - local label="${2:-$kms_port}" - local meta_file="$ART/kms-$label-meta.json" - local ca_file="$ART/kms-$label-ca.pem" - for i in $(seq 1 60); do - if curl -kfsS "https://127.0.0.1:$kms_port/prpc/KMS.GetMeta?json" -o "$meta_file"; then - jq -r '.ca_cert // .caCert // empty' "$meta_file" >"$ca_file" - if [[ -s "$ca_file" ]] && grep -q "BEGIN CERTIFICATE" "$ca_file"; then - # dstack-util treats KMS provider ID as the root CA - # SubjectPublicKeyInfo DER bytes (x509_parser public_key().raw). - openssl x509 -in "$ca_file" -pubkey -noout \ - | openssl pkey -pubin -outform DER \ - | od -An -tx1 -v \ - | tr -d ' \n' - echo - return 0 - fi - fi - sleep 2 - if [[ $((i % 10)) -eq 0 ]]; then echo "waiting for $label KMS GetMeta ca_cert..." >&2; fi - done - echo "failed to derive KMS key_provider_id from https://127.0.0.1:$kms_port/prpc/KMS.GetMeta" >&2 - exit 1 -} - -deploy_app() { - local name="$1" - local kms_port="$2" - local app_port="$3" - local key_provider_id - key_provider_id=$(kms_key_provider_id "$kms_port" "$name") - echo "$name key_provider_id=$key_provider_id" >&2 - cat >"$BASE/$name-compose.yaml" <<'YAML' -services: - smoke: - image: debian:bookworm-slim - command: sh -c 'echo SNP_APP_CONTAINER_STARTED; sleep 300' -YAML - python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" compose --docker-compose "$BASE/$name-compose.yaml" --name "$name" --kms --key-provider-id "$key_provider_id" --public-logs --public-sysinfo --no-instance-id --output "$BASE/$name.app-compose.json" | tee "$ART/$name-compose-create.txt" >&2 - jq --arg init_script "$DNS_INIT_SCRIPT" '.storage_fs="ext4" | .init_script=$init_script' "$BASE/$name.app-compose.json" >"$BASE/$name.app-compose.json.tmp" - mv "$BASE/$name.app-compose.json.tmp" "$BASE/$name.app-compose.json" - python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" deploy --name "$name" --compose "$BASE/$name.app-compose.json" --image "$IMAGE_NAME" --kms-url "https://10.0.2.2:$kms_port" --port "tcp:127.0.0.1:$app_port:8000" --vcpu 2 --memory 4096 --disk 20G | tee "$ART/$name-deploy.txt" >&2 - sed -n 's/Created VM with ID: //p' "$ART/$name-deploy.txt" | tail -1 -} - -if [[ "$RUN_STRICT_TCB_PROBE" = "1" && "$ALLOW_OUT_OF_DATE_TCB" = "1" ]]; then - echo "== Strict TCB probe: expect app GetAppKey denial on lab OutOfDate host ==" - STRICT_KMS_VM_ID=$(deploy_kms snp-smoke-kms-strict strict "$STRICT_KMS_HOST_PORT") - echo "STRICT_KMS_VM_ID=$STRICT_KMS_VM_ID" - wait_for_kms_metrics "$STRICT_KMS_VM_ID" "$STRICT_KMS_HOST_PORT" strict - STRICT_APP_VM_ID=$(deploy_app snp-smoke-app-strict "$STRICT_KMS_HOST_PORT" "$STRICT_APP_HOST_PORT") - echo "STRICT_APP_VM_ID=$STRICT_APP_VM_ID" - for i in $(seq 1 240); do - logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$STRICT_APP_VM_ID" -n 180 2>/dev/null || true) - kms_logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$STRICT_KMS_VM_ID" -n 220 2>/dev/null || true) - if { echo "$logs"; echo "$kms_logs"; } | grep -q "tcb_status is not allowed"; then - { echo "$logs"; echo "$kms_logs"; } | tee "$ART/strict-tcb-denial-log.txt" - echo "strict_tcb_probe=denied_as_expected" - break - fi - if { echo "$logs"; echo "$kms_logs"; } | grep -q "KDS collateral unavailable\|HTTP status client error"; then - { echo "$logs"; echo "$kms_logs"; } | tee "$ART/strict-tcb-kds-blocked-log.txt" - echo "strict_tcb_probe=blocked_by_kds_collateral" - break - fi - if echo "$logs" | grep -Eq "SNP_APP_CONTAINER_STARTED|Container dstack-smoke-1 Started"; then echo "$logs" | tee "$ART/strict-tcb-unexpected-success-log.txt"; echo "strict TCB probe unexpectedly reached app container"; exit 1; fi - sleep 2 - if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for strict APP denial..."; echo "$logs" | tail -60; echo "$kms_logs" | tail -60; fi - if [[ $i -eq 240 ]]; then echo "strict TCB probe did not reach expected denial"; { echo "$logs"; echo "$kms_logs"; } | tee "$ART/strict-tcb-timeout-log.txt"; exit 1; fi - done -fi - -echo "== KMS success run ==" -KMS_VM_ID=$(deploy_kms snp-smoke-kms "" "$KMS_HOST_PORT") -echo "KMS_VM_ID=$KMS_VM_ID" -wait_for_kms_metrics "$KMS_VM_ID" "$KMS_HOST_PORT" success -curl -kfsS "https://127.0.0.1:$KMS_HOST_PORT/metrics" | tee "$ART/kms-metrics-before-app.txt" - -APP_VM_ID=$(deploy_app snp-smoke-app "$KMS_HOST_PORT" "$APP_HOST_PORT") -echo "APP_VM_ID=$APP_VM_ID" - -for i in $(seq 1 240); do - logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$APP_VM_ID" -n 160 2>/dev/null || true) - if echo "$logs" | grep -Eq "SNP_APP_CONTAINER_STARTED|Container dstack-smoke-1 Started"; then echo "$logs" | tee "$ART/app-ready-log.txt"; echo "APP ready after ${i}s"; break; fi - if echo "$logs" | grep -q "Failed to get app key\|Failed to verify app\|Invalid mr_config\|amd sev-snp key release\|measurement mismatch\|App not allowed\|KMS self authorization failed\|KDS collateral unavailable\|HTTP status client error"; then echo "$logs" | tee "$ART/app-failure-log.txt"; exit 2; fi - sleep 2 - if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for APP..."; echo "$logs" | tail -60; fi - if [[ $i -eq 240 ]]; then echo "APP did not become ready"; echo "$logs" | tee "$ART/app-timeout-log.txt"; exit 1; fi -done - -curl -kfsS "https://127.0.0.1:$KMS_HOST_PORT/metrics" | tee "$ART/kms-metrics-after-app.txt" -python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" info "$KMS_VM_ID" --json | tee "$ART/kms-info.json" -python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" info "$APP_VM_ID" --json | tee "$ART/app-info.json" -python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$KMS_VM_ID" -n 200 | tee "$ART/kms-final-log.txt" || true -python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$APP_VM_ID" -n 200 | tee "$ART/app-final-log.txt" || true - -echo "== SNP E2E smoke success: $(date -Is) ==" -echo "Artifacts: $ART" diff --git a/dstack/tests/docs/kms-self-authorization.md b/dstack/tests/docs/kms-self-authorization.md index fbde050e9..d9e365fd9 100644 --- a/dstack/tests/docs/kms-self-authorization.md +++ b/dstack/tests/docs/kms-self-authorization.md @@ -293,7 +293,7 @@ Expected fields: - `device_id` - `mr_aggregated` - `os_image_hash` -- `attestation_mode` +- `tee_variant` Extract values: diff --git a/dstack/tests/e2e/attestation/Dockerfile b/dstack/tests/e2e/attestation/Dockerfile new file mode 100644 index 000000000..2d6bb0413 --- /dev/null +++ b/dstack/tests/e2e/attestation/Dockerfile @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# SPDX-License-Identifier: Apache-2.0 + +FROM rust:1.92-bookworm AS builder +WORKDIR /src +COPY . . +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=/src/target \ + cargo build --locked \ + -p dstack-tee-simulator \ + -p mock-attestation \ + -p dstack-util \ + -p dstack-verifier \ + -p dstack-mr \ + && mkdir -p /out \ + && cp target/debug/dstack-tee-simulator target/debug/dstack-mock-attestation \ + target/debug/dstack-util target/debug/dstack-verifier target/debug/dstack-mr /out/ \ + && strip /out/* + +FROM debian:bookworm-slim +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates curl fuse3 jq kmod openssl swtpm tpm2-tools xxd \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /out/ /usr/local/bin/ +COPY cc-eventlog/samples/tpm_eventlog.bin /usr/local/share/dstack/tpm_eventlog.bin +COPY verifier/fixtures/tdx-lite-getquote.json /usr/local/share/dstack/tdx-lite-getquote.json +COPY verifier/fixtures/sev-snp-attestation.json /usr/local/share/dstack/sev-snp-attestation.json +ENTRYPOINT ["/usr/local/bin/run-platform"] diff --git a/dstack/tests/e2e/attestation/README.md b/dstack/tests/e2e/attestation/README.md new file mode 100644 index 000000000..aeafd81f2 --- /dev/null +++ b/dstack/tests/e2e/attestation/README.md @@ -0,0 +1,147 @@ +# Attestation Docker Compose E2E + +This test runs the complete attestation flow in privileged Docker containers, +without starting a VM: + +```text +dstack-tee-simulator -> dstack-util attest -> dstack-verifier +``` + +The simulator and verifier use development trust anchors derived from the same +test seed. Verification still follows the normal production verification paths; +there is no mock-certificate bypass in the verifier. + +## Covered platforms + +The Compose suite runs these six cases: + +- dstack TDX, legacy attestation selector +- dstack TDX, lite attestation selector +- GCP TDX with a simulated vTPM +- AMD SEV-SNP +- AWS Nitro Enclave +- AWS NitroTPM + +Every case must produce all of the following results: + +```text +is_valid=true +quote_verified=true +event_log_verified=true +os_image_hash_verified=true +``` + +## Requirements + +- Linux host +- Docker with the Compose plugin +- Permission to run privileged containers +- Host kernel modules available through `/lib/modules` +- A kernel providing: + - FUSE and CUSE support + - `tpm_vtpm_proxy` (`CONFIG_TCG_VTPM_PROXY`) for the GCP vTPM case + +The script loads `tpm_vtpm_proxy` on demand. The Docker daemon therefore needs +access to a `/lib/modules` tree matching the running host kernel. + +## Run the complete suite + +From the `dstack/` directory: + +```bash +./tests/e2e/attestation/run.sh +``` + +The wrapper builds the local image, runs all six services sequentially, and +removes Compose resources afterward. + +## Run one case + +From this directory: + +```bash +cd tests/e2e/attestation +docker compose build +docker compose run --rm dstack-tdx-legacy +docker compose run --rm dstack-tdx-lite +docker compose run --rm gcp-tdx +docker compose run --rm amd-sev-snp +docker compose run --rm aws-nitro-enclave +docker compose run --rm aws-nitro-tpm +``` + +To clean up manually: + +```bash +docker compose down --remove-orphans +``` + +## What the test configures + +`run-platform.sh` creates a temporary sys-config inside each container. It +selects the simulator platform and supplies the corresponding image measurement +material: + +| Case | Image binding material | +| --- | --- | +| TDX legacy/lite | `tdx_measurement`, VM parameters, and the CCEL ACPI event digests | +| GCP TDX | `gcp_measurement` bound to the UKI Authenticode event and vTPM PCRs | +| AMD SEV-SNP | `sev_snp_measurement` and `mr_config`, bound to report `MEASUREMENT` and `HOST_DATA` | +| Nitro Enclave | NSM-signed PCR0/1/2 | +| NitroTPM | `aws_measurement` bound to NSM-signed PCR4/7/12 | + +The TDX simulator exposes the repository CCEL fixture through the normal TSM +configfs ABI. The GCP case exposes a TPM event log through +`/sys/kernel/security/tpm0/binary_bios_measurements` and replays it into +`swtpm`, so the quoted PCRs and event log remain consistent. + +The NitroTPM case exposes a kernel vTPM device backed by `swtpm`. Its proxy +implements AWS's NSM vendor command on the TPM wire protocol; the attestation +client therefore follows the same EK, salted session, authenticated NV buffer, +and vendor-command path used on real NitroTPM hardware. It does not expose or +fall back to `/dev/nsm`. + +Development root certificates are written as files and configured in +`verifier.toml`. The test explicitly enables +`insecure_allow_external_trust_anchors`; production defaults remain unchanged. + +## Troubleshooting + +### `modprobe: FATAL: Module tpm_vtpm_proxy not found` + +Install modules matching the running kernel or enable +`CONFIG_TCG_VTPM_PROXY` in the host kernel. + +### `/dev/nsm` is not created + +Ensure the host kernel has CUSE support and that the container is privileged. +The test creates the device node from `/sys/class/cuse/nsm/dev` because no udev +daemon runs inside the container. + +### FUSE/configfs permission errors + +Confirm that privileged containers are allowed. Rootless Docker is not +supported by this test because it mounts configfs/securityfs and creates device +nodes. + +### Inspecting a failure + +The per-container working directory is `/run/attestation-e2e`. Before the +container exits, the useful files are: + +```text +simulator.log +collateral.log +verifier.log +request.json +request.json.verification.json +verifier.toml +``` + +For interactive inspection, override the entrypoint and run the platform script +manually: + +```bash +docker compose run --rm --entrypoint bash gcp-tdx +TEE_PLATFORM=gcp-tdx /usr/local/bin/run-platform +``` diff --git a/dstack/tests/e2e/attestation/compose.yaml b/dstack/tests/e2e/attestation/compose.yaml new file mode 100644 index 000000000..e2ac964a6 --- /dev/null +++ b/dstack/tests/e2e/attestation/compose.yaml @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# SPDX-License-Identifier: Apache-2.0 + +x-attestation-test: &attestation-test + image: dstack-tests-attestation:local + build: + context: ../../.. + dockerfile: tests/e2e/attestation/Dockerfile + privileged: true + network_mode: bridge + volumes: + - /lib/modules:/lib/modules:ro + - ./run-platform.sh:/usr/local/bin/run-platform:ro + tmpfs: + - /run + - /dstack/.host-shared + - /tmp + +services: + dstack-tdx-legacy: + <<: *attestation-test + environment: { TEE_PLATFORM: dstack-tdx, TDX_ATTESTATION_VARIANT: legacy } + dstack-tdx-lite: + <<: *attestation-test + environment: { TEE_PLATFORM: dstack-tdx, TDX_ATTESTATION_VARIANT: lite } + gcp-tdx: + <<: *attestation-test + environment: { TEE_PLATFORM: dstack-gcp-tdx } + amd-sev-snp: + <<: *attestation-test + environment: { TEE_PLATFORM: dstack-amd-sev-snp } + aws-nitro-enclave: + <<: *attestation-test + environment: { TEE_PLATFORM: dstack-nitro-enclave } + aws-nitro-tpm: + <<: *attestation-test + environment: { TEE_PLATFORM: dstack-aws-nitro-tpm } diff --git a/dstack/tests/e2e/attestation/run-platform.sh b/dstack/tests/e2e/attestation/run-platform.sh new file mode 100755 index 000000000..b025b511c --- /dev/null +++ b/dstack/tests/e2e/attestation/run-platform.sh @@ -0,0 +1,268 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +: "${TEE_PLATFORM:?TEE_PLATFORM is required}" +SEED=7171717171717171717171717171717171717171717171717171717171717171 +REPORT_DATA=42424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242 +WORK=/run/attestation-e2e +SYS_CONFIG=/dstack/.host-shared/.sys-config.json +SIM_CONFIG=/dstack/.host-shared/.tee-simulator.json +mkdir -p "$WORK" /run/log/dstack +mount -t tmpfs -o mode=0755 tmpfs /sys/kernel/config +mkdir -p /sys/kernel/config/tsm/report + +VM_CONFIG='{}' +MR_CONFIG='{"version":3,"app_id":"","compose_hash":"","key_provider":"none"}' +GCP_TPM_REPLAY=null +AWS_PCR_REPLAY=null +if [[ "$TEE_PLATFORM" == dstack-tdx ]]; then + VM_CONFIG=$(jq -c --arg variant "${TDX_ATTESTATION_VARIANT:?}" \ + '.vm_config | fromjson | .tdx_attestation_variant = $variant' \ + /usr/local/share/dstack/tdx-lite-getquote.json) +elif [[ "$TEE_PLATFORM" == dstack-gcp-tdx ]]; then + UKI_HASH=9ab14a46f858662a89adc102d2a57a13f52f75c1769d65a4c34edbbfc8855f0f + printf '%s\n' "$UKI_HASH" > "$WORK/gcp-uki-hash" + dstack-mr gcp-measurement-cbor "$WORK/gcp-uki-hash" > "$WORK/measurement.gcp.cbor" + MEASUREMENT_HASH=$(sha256sum "$WORK/measurement.gcp.cbor" | cut -d' ' -f1) + printf '%s measurement.gcp.cbor\n' "$MEASUREMENT_HASH" > "$WORK/sha256sum.txt" + OS_IMAGE_HASH=$(sha256sum "$WORK/sha256sum.txt" | cut -d' ' -f1) + VM_CONFIG=$(jq -cn \ + --arg os "$OS_IMAGE_HASH" \ + --arg checksum "$(base64 -w0 "$WORK/sha256sum.txt")" \ + --arg measurement "$(base64 -w0 "$WORK/measurement.gcp.cbor")" \ + '{os_image_hash:$os,gcp_measurement:{checksum_file:$checksum,measurement:$measurement}}') + GCP_TPM_REPLAY=$(jq -cn \ + --arg event_log "$(base64 -w0 /usr/local/share/dstack/tpm_eventlog.bin)" \ + '{event_log:$event_log}') +elif [[ "$TEE_PLATFORM" == dstack-amd-sev-snp ]]; then + jq -r .attestation /usr/local/share/dstack/sev-snp-attestation.json | xxd -r -p > "$WORK/snp-fixture.bin" + dstack-util attest-json --input "$WORK/snp-fixture.bin" --output "$WORK/snp-fixture.json" + VM_CONFIG=$(jq -c '.config | fromjson' "$WORK/snp-fixture.json") + MR_CONFIG=$(jq -r .mr_config <<<"$VM_CONFIG") +elif [[ "$TEE_PLATFORM" == dstack-nitro-enclave ]]; then + for index in 0 1 2; do + printf 'dstack-tee-simulator/nsm/pcr/%s' "$index" | + sha384sum | cut -d' ' -f1 > "$WORK/pcr$index" + done + OS_IMAGE_HASH=$( + cat "$WORK/pcr0" "$WORK/pcr1" "$WORK/pcr2" | + xxd -r -p | sha256sum | cut -d' ' -f1 + ) + VM_CONFIG=$(jq -cn --arg os "$OS_IMAGE_HASH" '{os_image_hash:$os}') +elif [[ "$TEE_PLATFORM" == dstack-aws-nitro-tpm ]]; then + ZERO_PCR=$(printf '00%.0s' $(seq 1 48)) + printf '%s' "$ZERO_PCR" > "$WORK/pcr4" + cp "$WORK/pcr4" "$WORK/pcr7" + cp "$WORK/pcr4" "$WORK/pcr12" + dstack-mr aws-measurement-cbor "$ZERO_PCR" "$ZERO_PCR" "$ZERO_PCR" \ + > "$WORK/measurement.aws.cbor" + MEASUREMENT_HASH=$(sha256sum "$WORK/measurement.aws.cbor" | cut -d' ' -f1) + printf '%s measurement.aws.cbor\n' "$MEASUREMENT_HASH" > "$WORK/sha256sum.txt" + OS_IMAGE_HASH=$(sha256sum "$WORK/sha256sum.txt" | cut -d' ' -f1) + AWS_PCR_REPLAY=$(jq -cn --arg pcr "$ZERO_PCR" \ + '{version:1,events:[],pcr4:$pcr,pcr7:$pcr,pcr12:$pcr}') + VM_CONFIG=$(jq -cn \ + --arg os "$OS_IMAGE_HASH" \ + --arg checksum "$(base64 -w0 "$WORK/sha256sum.txt")" \ + --arg measurement "$(base64 -w0 "$WORK/measurement.aws.cbor")" \ + --argjson replay "$AWS_PCR_REPLAY" \ + '{os_image_hash:$os,aws_measurement:{checksum_file:$checksum,measurement:$measurement},aws_pcr_replay:$replay}') +fi + +cleanup() { + set +e + [[ -n "${SIM_PID:-}" ]] && kill "$SIM_PID" 2>/dev/null + [[ -n "${COLLATERAL_PID:-}" ]] && kill "$COLLATERAL_PID" 2>/dev/null + [[ -s /run/dstack/swtpm.pid ]] && kill "$(cat /run/dstack/swtpm.pid)" 2>/dev/null + fusermount3 -uz /sys/kernel/config/tsm/report 2>/dev/null + umount /sys/class/dmi/id/sys_vendor 2>/dev/null + umount /sys/class/dmi/id/product_name 2>/dev/null +} +trap cleanup EXIT + +cat > "$SYS_CONFIG" <<JSON +{ + "kms_urls": [], + "gateway_urls": [], + "collateral_urls": { + "pccs": "http://127.0.0.1:18088", + "amd_kds": "http://127.0.0.1:18088/vcek/v1" + }, + "docker_registry": null, + "host_api_url": null, + "mr_config": $(jq -Rn --arg value "$MR_CONFIG" '$value'), + "vm_config": $(jq -Rn --arg value "$VM_CONFIG" '$value') +} +JSON + +cat > "$SIM_CONFIG" <<JSON +{ + "platform": "$TEE_PLATFORM", + "mock_attestation_seed": "$SEED", + "collateral_base_url": "http://127.0.0.1:18088", + "mr_config": $(jq -Rn --arg value "$MR_CONFIG" '$value'), + "vm_config": $(jq -Rn --arg value "$VM_CONFIG" '$value'), + "gcp_tpm_replay": $GCP_TPM_REPLAY, + "aws_pcr_replay": $AWS_PCR_REPLAY +} +JSON + +# Reconstruct public roots and production-shaped collateral from the same seed. +dstack-mock-attestation serve \ + --listen 127.0.0.1:18088 \ + --config "$SIM_CONFIG" \ + --output "$WORK/roots" >"$WORK/collateral.log" 2>&1 & +COLLATERAL_PID=$! +for _ in $(seq 1 100); do + curl -fsS http://127.0.0.1:18088/tpm/aia/root.pem >/dev/null 2>&1 && break + sleep .05 +done +curl -fsS http://127.0.0.1:18088/tpm/aia/root.pem >/dev/null + +if [[ "$TEE_PLATFORM" == dstack-gcp-tdx || "$TEE_PLATFORM" == dstack-aws-nitro-tpm ]]; then + mount -t tmpfs -o mode=0755 tmpfs /sys/kernel/security + mkdir -p /sys/kernel/security/tpm0 + cp /usr/local/share/dstack/tpm_eventlog.bin /sys/kernel/security/tpm0/binary_bios_measurements + modprobe tpm_vtpm_proxy + if [[ ! -e /dev/vtpmx && -r /sys/class/misc/vtpmx/dev ]]; then + IFS=: read -r major minor < /sys/class/misc/vtpmx/dev + mknod /dev/vtpmx c "$major" "$minor" + fi + chmod 0666 /dev/vtpmx +fi + +dstack-tee-simulator --config "$SIM_CONFIG" >"$WORK/simulator.log" 2>&1 & +SIM_PID=$! +case "$TEE_PLATFORM" in + dstack-tdx|dstack-gcp-tdx|dstack-amd-sev-snp) + for _ in $(seq 1 200); do + [[ -e /sys/kernel/config/tsm/report/provider || -e /sys/kernel/config/tsm/report/com.intel.dcap/outblob ]] && break + sleep .05 + done + ;; + dstack-nitro-enclave) + for _ in $(seq 1 200); do + if [[ ! -e /dev/nsm && -r /sys/class/cuse/nsm/dev ]]; then + IFS=: read -r major minor < /sys/class/cuse/nsm/dev + mknod /dev/nsm c "$major" "$minor" + fi + [[ -e /dev/nsm ]] && break + sleep .05 + done + if [[ ! -e /dev/nsm ]]; then + echo "NSM device node was not created" >&2 + find /sys -path '*nsm*' -o -path '*cuse*' 2>/dev/null | head -100 >&2 + cat "$WORK/simulator.log" >&2 + exit 1 + fi + ;; + dstack-aws-nitro-tpm) + for _ in $(seq 1 200); do + [[ -e /dev/tpm0 || -e /dev/tpmrm0 ]] && break + sleep .05 + done + if [[ ! -e /dev/tpm0 && ! -e /dev/tpmrm0 ]]; then + echo "NitroTPM device node was not created" >&2 + cat "$WORK/simulator.log" >&2 + exit 1 + fi + ;; +esac +if ! kill -0 "$SIM_PID" 2>/dev/null; then + echo "[$TEE_PLATFORM] simulator failed:" >&2 + cat "$WORK/simulator.log" >&2 + exit 1 +fi +echo "platform=$TEE_PLATFORM dmi_vendor=$(cat /sys/class/dmi/id/sys_vendor 2>/dev/null || true) dmi_product=$(cat /sys/class/dmi/id/product_name 2>/dev/null || true) nsm=$(test -e /dev/nsm && echo yes || echo no)" + +export DCAP_TDX_RTMR_SYSFS_PATH=/sys/kernel/config/tsm/report/com.intel.dcap/measurements +export DSTACK_CCEL_FILE=/sys/kernel/config/tsm/report/com.intel.dcap/ccel +if ! ATTESTATION_HEX=$(dstack-util attest --report-data "$REPORT_DATA" --hex); then + cat "$WORK/simulator.log" >&2 + exit 1 +fi +jq -n --arg attestation "$ATTESTATION_HEX" \ + '{attestation: $attestation, debug: true}' > "$WORK/request.json" + +cat > "$WORK/verifier.toml" <<EOF_CONFIG +address = "127.0.0.1" +port = 8080 +image_cache_dir = "$WORK/image-cache" +image_download_url = "http://127.0.0.1:9/mr_{OS_IMAGE_HASH}.tar.gz" +image_download_timeout_secs = 1 + +[attestation] +insecure_allow_external_trust_anchors = true + +[attestation.urls] +pccs = "http://127.0.0.1:18088" +amd_kds = "http://127.0.0.1:18088/vcek/v1" + +[attestation.root_ca] +tdx = "$WORK/roots/tdx-root-ca.pem" +gcp_tpm = "$WORK/roots/tpm-root-ca.pem" +aws_nitro_enclave = "$WORK/roots/nsm-root-ca.pem" +aws_nitro_tpm = "$WORK/roots/nsm-root-ca.pem" +sev_snp_milan = "$WORK/roots/sev-snp-root-ca.pem" +sev_snp_genoa = "$WORK/roots/sev-snp-root-ca.pem" +sev_snp_turin = "$WORK/roots/sev-snp-root-ca.pem" +EOF_CONFIG + +set +e +dstack-verifier --config "$WORK/verifier.toml" --verify "$WORK/request.json" \ + >"$WORK/verifier.log" 2>&1 +VERIFIER_RC=$? +set -e +if [[ ! -s "$WORK/request.json.verification.json" ]]; then + cat "$WORK/verifier.log" >&2 + exit 1 +fi +if (( VERIFIER_RC != 0 )); then + cat "$WORK/verifier.log" >&2 + exit "$VERIFIER_RC" +fi +cat "$WORK/request.json.verification.json" +jq -e '.details.quote_verified == true' "$WORK/request.json.verification.json" >/dev/null +jq -e '.is_valid == true' "$WORK/request.json.verification.json" >/dev/null +jq -e '.details.os_image_hash_verified == true' "$WORK/request.json.verification.json" >/dev/null +jq -e '.details.event_log_verified == true' "$WORK/request.json.verification.json" >/dev/null +cp "$WORK/request.json.verification.json" "$WORK/development-root-verification.json" +echo '{"development_root_accepted":true}' + +# The same simulator evidence must fail against the verifier's built-in +# production roots. Keep the mock collateral endpoints so this assertion tests +# the trust-anchor boundary rather than network or collateral availability. +cat > "$WORK/production-verifier.toml" <<EOF_CONFIG +address = "127.0.0.1" +port = 8080 +image_cache_dir = "$WORK/production-image-cache" +image_download_url = "http://127.0.0.1:9/mr_{OS_IMAGE_HASH}.tar.gz" +image_download_timeout_secs = 1 + +[attestation.urls] +pccs = "http://127.0.0.1:18088" +amd_kds = "http://127.0.0.1:18088/vcek/v1" +EOF_CONFIG +rm -f "$WORK/request.json.verification.json" +set +e +dstack-verifier --config "$WORK/production-verifier.toml" --verify "$WORK/request.json" \ + >"$WORK/production-verifier.log" 2>&1 +PRODUCTION_VERIFIER_RC=$? +set -e +if [[ ! -s "$WORK/request.json.verification.json" ]]; then + cat "$WORK/production-verifier.log" >&2 + echo "production-root verifier did not emit a verification result" >&2 + exit 1 +fi +cat "$WORK/request.json.verification.json" +if (( PRODUCTION_VERIFIER_RC == 0 )) || ! jq -e '.is_valid == false' "$WORK/request.json.verification.json" >/dev/null; then + cat "$WORK/production-verifier.log" >&2 + echo "simulator evidence unexpectedly passed production-root verification" >&2 + exit 1 +fi +echo '{"production_root_rejected":true}' +echo "[$TEE_PLATFORM${TDX_ATTESTATION_VARIANT:+/$TDX_ATTESTATION_VARIANT}] dstack-util -> verifier trust-root isolation E2E passed" diff --git a/dstack/tests/e2e/attestation/run.sh b/dstack/tests/e2e/attestation/run.sh new file mode 100755 index 000000000..21ac55a22 --- /dev/null +++ b/dstack/tests/e2e/attestation/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail +cd "$(dirname "$0")" +docker compose build +for platform in dstack-tdx-legacy dstack-tdx-lite gcp-tdx amd-sev-snp aws-nitro-enclave aws-nitro-tpm; do + echo "=== $platform ===" + docker compose run --rm "$platform" +done +docker compose down --remove-orphans diff --git a/dstack/test-scripts/get-app-key.sh b/dstack/tests/get-app-key.sh similarity index 76% rename from dstack/test-scripts/get-app-key.sh rename to dstack/tests/get-app-key.sh index 7285a4fbe..7fbff4fa4 100755 --- a/dstack/test-scripts/get-app-key.sh +++ b/dstack/tests/get-app-key.sh @@ -11,5 +11,5 @@ URL=https://localhost:8043/prpc/KMS.GetAppKey if [ -z "$CERT_FILE" ]; then curl -s --cacert ${CERT_DIR}/ca.cert ${URL} else - curl -vv --cacert ${CERT_DIR}/ca.cert --cert ${CERT_FILE} --key ${KEY_FILE} ${URL} + curl -vv --cacert "${CERT_DIR}/ca.cert" --cert "${CERT_FILE}" --key "${KEY_FILE}" "${URL}" fi diff --git a/dstack/test-scripts/inspect-cert.sh b/dstack/tests/inspect-cert.sh similarity index 77% rename from dstack/test-scripts/inspect-cert.sh rename to dstack/tests/inspect-cert.sh index 6ecb93389..da6fd62dc 100755 --- a/dstack/test-scripts/inspect-cert.sh +++ b/dstack/tests/inspect-cert.sh @@ -3,4 +3,4 @@ # SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network> # SPDX-License-Identifier: Apache-2.0 -openssl x509 -text -noout -in $1 +openssl x509 -text -noout -in "$1" diff --git a/dstack/tpm-attest/src/lib.rs b/dstack/tpm-attest/src/lib.rs index 7105af79d..a0f3046a2 100644 --- a/dstack/tpm-attest/src/lib.rs +++ b/dstack/tpm-attest/src/lib.rs @@ -48,7 +48,7 @@ const AWS_NITRO_PCRS: [u32; 5] = [4, 7, 8, 12, 14]; pub fn dstack_pcr_policy_for_platform(platform: Platform) -> Result<PcrSelection> { match platform { - Platform::Gcp => Ok(dstack_pcr_policy()), + Platform::Dstack | Platform::Gcp => Ok(dstack_pcr_policy()), Platform::AwsEc2 => Ok(PcrSelection::sha384(&AWS_NITRO_PCRS)), _ => bail!("TPM local key provider is not supported on {platform:?}"), } @@ -367,6 +367,16 @@ mod tests { let policy = dstack_pcr_policy_for_platform(Platform::AwsEc2).unwrap(); assert_eq!(policy.to_arg(), "sha384:4,7,8,12,14"); } + + #[test] + fn dstack_platform_uses_development_tpm_policy() { + assert_eq!( + dstack_pcr_policy_for_platform(Platform::Dstack) + .unwrap() + .to_arg(), + "sha256:0,2,14" + ); + } } mod gcp_ak; diff --git a/dstack/tpm-qvl/src/lib.rs b/dstack/tpm-qvl/src/lib.rs index 6a260ab18..973b990a1 100644 --- a/dstack/tpm-qvl/src/lib.rs +++ b/dstack/tpm-qvl/src/lib.rs @@ -37,6 +37,48 @@ pub fn get_root_ca(platform: Platform) -> Result<&'static str> { } } +/// Verifies TPM quotes against a caller-selected trust anchor. +/// +/// This follows `dcap_qvl::verify::QuoteVerifier`: [`Self::new_prod`] selects +/// the bundled platform root and [`Self::new`] accepts a PEM root CA. +#[derive(Debug, Clone)] +pub struct QuoteVerifier { + root_ca_pem: String, +} + +impl QuoteVerifier { + pub fn new(root_ca_pem: impl Into<String>) -> Self { + Self { + root_ca_pem: root_ca_pem.into(), + } + } + + pub fn new_prod(platform: Platform) -> Result<Self> { + Ok(Self::new(get_root_ca(platform)?)) + } + + pub fn root_ca_pem(&self) -> &str { + &self.root_ca_pem + } + + pub fn verify( + &self, + quote: &tpm_types::TpmQuote, + collateral: &QuoteCollateral, + ) -> Result<verify::VerifiedReport, VerificationError> { + verify::verify_quote_with_ca(quote, collateral, &self.root_ca_pem) + } + + #[cfg(feature = "crl-download")] + pub async fn fetch_and_verify( + &self, + quote: &tpm_types::TpmQuote, + ) -> Result<verify::VerifiedReport> { + let collateral = collateral::get_collateral(quote, &self.root_ca_pem).await?; + self.verify(quote, &collateral).map_err(Into::into) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct QuoteCollateral { /// Intermediate certificate chain (PEM format) from device diff --git a/dstack/tpm2/src/commands.rs b/dstack/tpm2/src/commands.rs index a64dc3a49..85f266a65 100644 --- a/dstack/tpm2/src/commands.rs +++ b/dstack/tpm2/src/commands.rs @@ -8,6 +8,7 @@ use anyhow::{Context, Result}; use std::collections::HashSet; +use std::io::{Read, Write}; use tracing::debug; use super::constants::*; @@ -32,6 +33,13 @@ impl TpmContext { Ok(Self { device }) } + /// Create a TPM context over an already connected byte stream. + pub fn from_stream(stream: impl Read + Write + 'static, name: impl Into<String>) -> Self { + Self { + device: TpmDevice::from_stream(stream, name), + } + } + /// Get the device path pub fn device_path(&self) -> &str { self.device.path() diff --git a/dstack/tpm2/src/device.rs b/dstack/tpm2/src/device.rs index c6cce8a63..32cea0aa2 100644 --- a/dstack/tpm2/src/device.rs +++ b/dstack/tpm2/src/device.rs @@ -18,8 +18,17 @@ use super::marshal::*; const TPM_MAX_COMMAND_SIZE: usize = 4096; /// TPM device handle +trait ReadWrite: Read + Write {} + +impl<T: Read + Write> ReadWrite for T {} + +enum TpmTransport { + Device(File), + Stream(Box<dyn ReadWrite>), +} + pub struct TpmDevice { - file: File, + transport: TpmTransport, path: String, } @@ -36,11 +45,19 @@ impl TpmDevice { .with_context(|| format!("failed to open TPM device: {}", device_path))?; Ok(Self { - file, + transport: TpmTransport::Device(file), path: device_path.to_string(), }) } + /// Create a TPM device over an already connected byte stream. + pub fn from_stream(stream: impl Read + Write + 'static, name: impl Into<String>) -> Self { + Self { + transport: TpmTransport::Stream(Box::new(stream)), + path: name.into(), + } + } + /// Detect and open the default TPM device pub fn detect() -> Result<Self> { if Path::new("/dev/tpmrm0").exists() { @@ -57,22 +74,41 @@ impl TpmDevice { &self.path } - /// Send a command to the TPM and receive the response + /// Send a command to the TPM and receive the response. pub fn transmit(&mut self, command: &[u8]) -> Result<Vec<u8>> { - // Write command - self.file - .write_all(command) - .context("failed to write TPM command")?; - - // Read response - let mut response = vec![0u8; TPM_MAX_COMMAND_SIZE]; - let n = self - .file - .read(&mut response) - .context("failed to read TPM response")?; - - response.truncate(n); - Ok(response) + match &mut self.transport { + TpmTransport::Device(file) => { + file.write_all(command) + .context("failed to write TPM command")?; + let mut response = vec![0u8; TPM_MAX_COMMAND_SIZE]; + let size = file + .read(&mut response) + .context("failed to read TPM response")?; + response.truncate(size); + Ok(response) + } + TpmTransport::Stream(stream) => { + stream + .write_all(command) + .context("failed to write TPM command")?; + let mut header = [0u8; 10]; + stream + .read_exact(&mut header) + .context("failed to read TPM response header")?; + let response_size = + u32::from_be_bytes([header[2], header[3], header[4], header[5]]) as usize; + if !(header.len()..=TPM_MAX_COMMAND_SIZE).contains(&response_size) { + bail!("invalid TPM response size: {response_size}"); + } + let mut response = Vec::with_capacity(response_size); + response.extend_from_slice(&header); + response.resize(response_size, 0); + stream + .read_exact(&mut response[header.len()..]) + .context("failed to read TPM response body")?; + Ok(response) + } + } } /// Execute a TPM command and parse the response @@ -275,6 +311,17 @@ impl TpmResponse { } } + /// Serialize this response back to TPM wire format. + pub fn to_bytes(&self) -> Vec<u8> { + let size = 10 + self.data.len(); + let mut response = Vec::with_capacity(size); + response.extend_from_slice(&self.tag.to_u16().to_be_bytes()); + response.extend_from_slice(&(size as u32).to_be_bytes()); + response.extend_from_slice(&self.response_code.to_be_bytes()); + response.extend_from_slice(&self.data); + response + } + /// Get a response buffer for parsing the data pub fn data_buffer(&self) -> ResponseBuffer<'_> { ResponseBuffer::new(&self.data) @@ -290,10 +337,104 @@ impl TpmResponse { } } +/// Add a command to a successful TPM_CAP_COMMANDS response. +/// +/// Responses for other capabilities, TPM errors, and paginated command lists +/// are returned unchanged. +pub fn add_command_capability(response: &[u8], command_code: u32) -> Result<Vec<u8>> { + let mut response = TpmResponse::parse(response)?; + if !response.is_success() { + return Ok(response.to_bytes()); + } + let mut buf = response.data_buffer(); + let more_data = buf.get_u8()? != 0; + let capability = buf.get_u32()?; + if capability != TpmCap::Commands as u32 || more_data { + return Ok(response.to_bytes()); + } + let count = buf.get_u32()? as usize; + let mut attributes = Vec::with_capacity(count + 1); + for _ in 0..count { + attributes.push(buf.get_u32()?); + } + if attributes.contains(&command_code) { + return Ok(response.to_bytes()); + } + const COMMAND_INDEX_AND_VENDOR_MASK: u32 = 0x2000_ffff; + let sort_key = command_code & COMMAND_INDEX_AND_VENDOR_MASK; + let insertion = attributes + .iter() + .position(|attributes| attributes & COMMAND_INDEX_AND_VENDOR_MASK > sort_key) + .unwrap_or(attributes.len()); + attributes.insert(insertion, command_code); + + let mut data = Vec::with_capacity(9 + attributes.len() * 4); + data.push(u8::from(more_data)); + data.extend_from_slice(&(TpmCap::Commands as u32).to_be_bytes()); + data.extend_from_slice(&(attributes.len() as u32).to_be_bytes()); + for attributes in attributes { + data.extend_from_slice(&attributes.to_be_bytes()); + } + response.data = data; + Ok(response.to_bytes()) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn stream_transport_reads_a_framed_response() { + use std::os::unix::net::UnixStream; + use std::thread; + + let (client, mut server) = UnixStream::pair().unwrap(); + let command = TpmCommand::new(TpmCc::GetRandom).finalize(); + let command_len = command.len(); + let server_thread = thread::spawn(move || { + let mut received = vec![0; command_len]; + server.read_exact(&mut received).unwrap(); + let response = TpmResponse { + tag: TpmSt::NoSessions, + response_code: 0, + data: vec![0, 0], + } + .to_bytes(); + server.write_all(&response[..10]).unwrap(); + server.write_all(&response[10..]).unwrap(); + }); + + let mut device = TpmDevice::from_stream(client, "test stream"); + let response = device.execute(&command).unwrap(); + assert!(response.is_success()); + assert_eq!(response.data, [0, 0]); + server_thread.join().unwrap(); + } + + #[test] + fn adds_a_vendor_command_to_command_capabilities() { + let response = TpmResponse { + tag: TpmSt::NoSessions, + response_code: 0, + data: { + let mut data = vec![0]; + data.extend_from_slice(&(TpmCap::Commands as u32).to_be_bytes()); + data.extend_from_slice(&1u32.to_be_bytes()); + data.extend_from_slice(&0x2000_1000u32.to_be_bytes()); + data + }, + } + .to_bytes(); + let response = add_command_capability(&response, 0x2000_0001).unwrap(); + let response = TpmResponse::parse(&response).unwrap(); + let mut data = response.data_buffer(); + assert_eq!(data.get_u8().unwrap(), 0); + assert_eq!(data.get_u32().unwrap(), TpmCap::Commands as u32); + assert_eq!(data.get_u32().unwrap(), 2); + assert_eq!(data.get_u32().unwrap(), 0x2000_0001); + assert_eq!(data.get_u32().unwrap(), 0x2000_1000); + } + #[test] fn test_command_builder() { let mut cmd = TpmCommand::new(TpmCc::GetRandom); diff --git a/dstack/tpm2/src/lib.rs b/dstack/tpm2/src/lib.rs index ab5db337e..ac670d9a1 100644 --- a/dstack/tpm2/src/lib.rs +++ b/dstack/tpm2/src/lib.rs @@ -44,6 +44,6 @@ pub use constants::*; pub use types::*; // Re-export device for advanced usage -pub use device::{TpmCommand, TpmDevice, TpmResponse}; +pub use device::{add_command_capability, TpmCommand, TpmDevice, TpmResponse}; pub use marshal::{CommandBuffer, Marshal, ResponseBuffer, Unmarshal}; pub use session::AuthSession; diff --git a/dstack/vendor/README.md b/dstack/vendor/README.md new file mode 100644 index 000000000..27244df8e --- /dev/null +++ b/dstack/vendor/README.md @@ -0,0 +1,42 @@ +# Vendored crates + +Third-party crates carried in-tree because the published release does not build +for a target dstack ships. Each one is a copy of a specific crates.io release +plus a named upstream patch, so the diff against the release is small enough to +review and the directory can be deleted once upstream publishes the fix. + +## ktls 6.0.2 + +**Why:** `ktls` 6.0.2 builds `libc::cmsghdr` and `libc::msghdr` with struct +literal syntax. That works on glibc, where those structs have exactly the +members the crate names, but musl's ABI declares `msg_iovlen` and +`msg_controllen` as `int`/`socklen_t` followed by explicit padding, and `libc` +models that faithfully with private `__pad1`/`__pad2` members. A struct literal +cannot name a private field, so the crate does not compile for +`x86_64-unknown-linux-musl` -- which is the target the gateway CVM app image is +built for: + +``` +error[E0063]: missing field `__pad1` in initializer of `cmsghdr` +error: cannot construct `msghdr` with struct literal syntax due to private fields +``` + +This is a permanent property of musl's ABI, not a transient `libc` bug, and +6.0.2 is the latest published release, so there is no version to bump to. + +**Patch:** [rustls/ktls#70](https://github.com/rustls/ktls/pull/70) ("build: add +musl support"), open upstream since 2026-05-04. Both structs are built field by +field from `std::mem::zeroed()` instead of with a literal. The patch is applied +to the 6.0.2 release rather than to upstream `main`, so this copy differs from +crates.io only in those two functions; each hunk is marked with a +`dstack patch (rustls/ktls#70)` comment. + +**Other differences from the crates.io package:** `[dev-dependencies]` are +dropped (upstream's test suite is not run here and would otherwise pull `rcgen`, +`test-case` and friends into this workspace's lock file), and `publish = false` +is set. `LICENSE-MIT` and `LICENSE-APACHE` are copied from the upstream +repository, which the crates.io package does not ship. + +**Remove this when:** upstream merges #70 and publishes a release containing it. +At that point delete `vendor/ktls`, drop the workspace member, and point +`ktls` in the root `Cargo.toml` back at the crates.io version. diff --git a/dstack/vendor/ktls/CHANGELOG.md b/dstack/vendor/ktls/CHANGELOG.md new file mode 100644 index 000000000..bfed11c2c --- /dev/null +++ b/dstack/vendor/ktls/CHANGELOG.md @@ -0,0 +1,110 @@ +# Changelog + +## [6.0.2](https://github.com/rustls/ktls/compare/ktls-v6.0.1...ktls-v6.0.2) - 2025-04-07 + +### Other + +- Make CryptoInfo public to enable asynchronous kTLS setup via io-uring or any mechanism other + than ktls's default synchronous `setsockopt` call, cf. <https://github.com/rustls/ktls/pull/54> + +## [6.0.1](https://github.com/rustls/ktls/compare/ktls-v6.0.0...ktls-v6.0.1) - 2024-09-26 + +### Other + +- Standardize READMEs a little +- Turn ktls into a workspace + +## [6.0.0](https://github.com/bearcove/ktls/compare/v5.0.0...v6.0.0) - 2024-08-14 + +### Added +- [**breaking**] Upgrade other dependencies +- [**breaking**] Upgrade to tokio-rustls 0.26.0, remove unused trait +- Migrate from ktls-recvmsg to nix 0.28 +- Switch from futures to futures_util +- Add aws-lc-rs as an alias for aws_lc_rs feature +- Upgrade dependencies + +### Fixed +- Upgrade to Rust 1.80.1 +- Remove nix stuff + +### Other +- Bump rustls from 0.23.2 to 0.23.5 +- Remove Cargo.lock from .gitignore +- Upgrade ktls-sys +- Ignore .DS_Store files +- Bump tokio-rustls +- make aws_lc_rs and ring mutually exclusive for ktls +- More aws_lc_rs support +- Start adding support for aws_lc_rs + +## [5.0.0](https://github.com/hapsoc/ktls/compare/v4.0.0...v5.0.0) - 2024-03-11 + +### Added +- [**breaking**] Upgrade to rustls 0.22.2 +- [**breaking**] Upgrade to tokio-rustls 0.25.0 + +### Other +- Get rid of constrandom (it caused 'wrong type' compile error on Rust stable) +- Print backtraces in CI +- Add nix flake to have the toolchain everywhere +- Re-add Cargo.lock as per updated best practices +- Disable incremental compilation +- Remove token +- Install missing tools +- Use the sccache action +- Just straight up try running it on GitHub-hosted runners + +## [4.0.0](https://github.com/hapsoc/ktls/compare/v3.0.2...v4.0.0) - 2023-10-08 + +### Fixed +- [**breaking**] Remove drained_remaining public method + +### Other +- Add more test coverage +- Remove more explicit libc::close calls +- Clarifies what this.inner.poll_shutdown does +- Improve integration tests: try reading/writing after close, catch errors from both sides +- Don't forget to close fd on writer side +- Simplify/clarify code around alerts +- Use enums to 'parse' TLS alerts +- Depend on ktls-recvmsg v0.1.3 +- Remove panic, ktls may send unfinished alert msg +- assert instead of asser_eq +- Adding edge case in integration test for session shutdown +- Properly handle critical alerts +- Add crates.io badge +- Use Rust stable for tests + +## [3.0.2](https://github.com/hapsoc/ktls/compare/v3.0.1...v3.0.2) - 2023-10-02 + +### Other +- Create FUNDING.yml +- Upgrade rcgen to 0.11.3 +- Upgrade dependencies + +## 3.0.1 (unreleased) + +Fix test suite (follow rustls' `ClientConfig::enable_tickets` transition to +`ClientConfig::resumption`). + +## 3.0.0 (2023-06-14) (yanked) + +Upgrade to tokio-rustls 0.24.1 + +## 2.0.0 (2023-03-29) + +Comes with a bunch of breaking changes, necessary to address some issues. + +Essentially, the rustls stream wasn't being drained properly in +`config_ktls_{client,server}`. Doing this properly required introducing +`CorkStream`, which is TLS-framing-aware. + +As a result, `config_ktls_*` functions now take a `TlsStream<CorkStream<IO>>` +(where `IO` is typically `TcpStream`), and are async, since to properly drain we +might need to read till the end of the last TLS messages rustls has partially +buffered. + +## 1.0.1 (2022-10-21) + +Initial release. diff --git a/dstack/vendor/ktls/Cargo.toml b/dstack/vendor/ktls/Cargo.toml new file mode 100644 index 000000000..7b402c87e --- /dev/null +++ b/dstack/vendor/ktls/Cargo.toml @@ -0,0 +1,41 @@ +# Vendored copy of ktls 6.0.2 -- see ../README.md for why and when to drop it. +# +# Transcribed from the crates.io package manifest. Two deliberate differences: +# the dev-dependencies are dropped (upstream's test suite is not run here and +# would pull rcgen, test-case and friends into this workspace's lock file), and +# the version carries a `+dstack` build metadata suffix so a stray crates.io +# resolution is impossible to confuse with this copy. +[package] +name = "ktls" +version = "6.0.2" +edition = "2021" +rust-version = "1.75" +authors = ["Amos Wenger <amos@bearcove.net>"] +description = "Configures kTLS for tokio-rustls client and server connections." +documentation = "https://docs.rs/ktls" +readme = "README.md" +license = "MIT OR Apache-2.0" +repository = "https://github.com/rustls/ktls" +publish = false + +[dependencies] +futures-util = "0.3.30" +ktls-sys = "1.0.1" +libc = { version = "0.2.155", features = ["const-extern-fn"] } +memoffset = "0.9.1" +nix = { version = "0.29.0", features = ["socket", "uio", "net"] } +num_enum = "0.7.3" +pin-project-lite = "0.2.14" +rustls = { version = "0.23.12", default-features = false } +smallvec = "1.13.2" +thiserror = "1.0.63" +tokio = { version = "1.39.2", features = ["net", "macros", "io-util"] } +tokio-rustls = { version = "0.26.0", default-features = false } +tracing = "0.1.40" + +[features] +default = ["aws_lc_rs", "tls12"] +aws-lc-rs = ["aws_lc_rs"] +aws_lc_rs = ["rustls/aws_lc_rs", "tokio-rustls/aws_lc_rs"] +ring = ["rustls/ring", "tokio-rustls/ring"] +tls12 = ["rustls/tls12", "tokio-rustls/tls12"] diff --git a/dstack/vendor/ktls/LICENSE-APACHE b/dstack/vendor/ktls/LICENSE-APACHE new file mode 100644 index 000000000..c98d27d4f --- /dev/null +++ b/dstack/vendor/ktls/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/LICENSE-2.0 + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/dstack/vendor/ktls/LICENSE-MIT b/dstack/vendor/ktls/LICENSE-MIT new file mode 100644 index 000000000..969d061e8 --- /dev/null +++ b/dstack/vendor/ktls/LICENSE-MIT @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/dstack/vendor/ktls/README.md b/dstack/vendor/ktls/README.md new file mode 100644 index 000000000..59c8897c1 --- /dev/null +++ b/dstack/vendor/ktls/README.md @@ -0,0 +1,17 @@ +[![test pipeline](https://github.com/hapsoc/ktls/actions/workflows/test.yml/badge.svg)](https://github.com/hapsoc/ktls/actions/workflows/test.yml?query=branch%3Amain) +[![Coverage Status (codecov.io)](https://codecov.io/gh/hapsoc/ktls/branch/main/graph/badge.svg)](https://codecov.io/gh/hapsoc/ktls/) +[![Crates.io](https://img.shields.io/crates/v/ktls)](https://crates.io/crates/ktls) +[![license: MIT/Apache-2.0](https://img.shields.io/badge/license-MIT%2FApache--2.0-blue.svg)](LICENSE-MIT) + +# ktls + +Configures kTLS ([kernel TLS +offload](https://www.kernel.org/doc/html/latest/networking/tls-offload.html)) +for any type that implements `AsRawFd`, given a rustls `ServerConnection`. + +## License + +This project is primarily distributed under the terms of both the MIT license +and the Apache License (Version 2.0). + +See [LICENSE-APACHE](LICENSE-APACHE) and [LICENSE-MIT](LICENSE-MIT) for details. diff --git a/dstack/vendor/ktls/src/async_read_ready.rs b/dstack/vendor/ktls/src/async_read_ready.rs new file mode 100644 index 000000000..b8d9a21d6 --- /dev/null +++ b/dstack/vendor/ktls/src/async_read_ready.rs @@ -0,0 +1,12 @@ +use std::{io, task}; + +pub trait AsyncReadReady { + /// cf. https://docs.rs/tokio/latest/tokio/net/struct.TcpStream.html#method.poll_read_ready + fn poll_read_ready(&self, cx: &mut task::Context<'_>) -> task::Poll<io::Result<()>>; +} + +impl AsyncReadReady for tokio::net::TcpStream { + fn poll_read_ready(&self, cx: &mut task::Context<'_>) -> task::Poll<io::Result<()>> { + tokio::net::TcpStream::poll_read_ready(self, cx) + } +} diff --git a/dstack/vendor/ktls/src/cork_stream.rs b/dstack/vendor/ktls/src/cork_stream.rs new file mode 100644 index 000000000..9d8271ec0 --- /dev/null +++ b/dstack/vendor/ktls/src/cork_stream.rs @@ -0,0 +1,214 @@ +use std::{io, pin::Pin, task}; + +use rustls::internal::msgs::codec::Codec; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +use crate::AsyncReadReady; + +enum State { + ReadHeader { header_buf: [u8; 5], offset: usize }, + ReadPayload { msg_size: usize, offset: usize }, + // we encountered EOF while reading, or saw an invalid header and we're just + // passing reads through without doing any sort of processing now. + Passthrough, +} + +/// This is a wrapper that reads TLS message headers so it knows when to start +/// doing empty reads at the message boundary when "draining" a rustls +/// connection before setting up kTLS for it. +/// +/// The short explanation is: rustls might have buffered one or more +/// ApplicationData messages (the last one might even be partial) by the time +/// "connect" / "accept" returns. +/// +/// We not only need to pop messages rustls has already deframed (that's done in +/// a drain function elsewhere), but also let rustls finish reading and +/// deframing any partial message it may have already buffered. +/// +/// Because this wrapper is trying very hard not to do any error handling, if it +/// encounters anything that doesn't look like a TLS header (unknown type, +/// nonsensical size, unexpected EOF), it'll quite easily fall back to a +/// "passthrough" mode with no internal buffering, letting rustls take care +/// reporting any errors. +pub struct CorkStream<IO> { + pub io: IO, + // if true, causes empty reads at the message boudnary + pub corked: bool, + state: State, +} + +impl<IO> CorkStream<IO> { + pub fn new(io: IO) -> Self { + Self { + io, + corked: false, + state: State::ReadHeader { + header_buf: Default::default(), + offset: 0, + }, + } + } +} + +impl<IO> AsyncRead for CorkStream<IO> +where + IO: AsyncRead, +{ + #[inline] + fn poll_read( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> task::Poll<io::Result<()>> { + let this = unsafe { self.get_unchecked_mut() }; + let mut io = unsafe { Pin::new_unchecked(&mut this.io) }; + + let state = &mut this.state; + + loop { + match state { + State::ReadHeader { header_buf, offset } => { + if *offset == 0 && this.corked { + tracing::trace!( + "corked, returning empty read (but waking to prevent stalls)" + ); + cx.waker().wake_by_ref(); + return task::Poll::Ready(Ok(())); + } + + let left = header_buf.len() - *offset; + tracing::trace!("reading header, {left}/{} bytes left", header_buf.len()); + + { + let mut rest = ReadBuf::new(&mut header_buf[*offset..]); + tracing::trace!("reading header: doing i/o"); + futures_util::ready!(io.as_mut().poll_read(cx, &mut rest)?); + tracing::trace!("reading header: io was ready"); + *offset += rest.filled().len(); + if rest.filled().is_empty() { + // that's an unexpected EOF for sure, but let's have + // rustls deal with the error reporting shall we? + tracing::trace!( + "unexpected EOF: header cut short after {} bytes", + *offset + ); + buf.put_slice(&header_buf[..*offset]); + *state = State::Passthrough; + + return task::Poll::Ready(Ok(())); + } + tracing::trace!("read {} bytes off of header", rest.filled().len()); + } + + if *offset == 5 { + // TODO: handle cases where buffer has less than 5 bytes + // remaining. I (fasterthanlime) bet this never happens in + // practice since the rustls deframer uses `copy_within` to + // get rid of the part of the buffer it's already decoded. + assert!(buf.remaining() >= 5, "you found an edge case in ktls!"); + buf.put_slice(&header_buf[..]); + + match decode_header(*header_buf) { + Some((typ, version, len)) => { + tracing::trace!( + "read header: typ={typ:?}, version={version:?}, len={len}" + ); + *state = State::ReadPayload { + msg_size: len as usize, + offset: 0, + }; + } + None => { + // we encountered an invalid header, let's bail out + tracing::warn!("encountered invalid header, bailing out"); + *state = State::Passthrough; + } + } + + return task::Poll::Ready(Ok(())); + } else { + // keep trying + } + } + State::ReadPayload { msg_size, offset } => { + let rest = *msg_size - *offset; + + let just_read = { + let mut rest = buf.take(rest); + futures_util::ready!(io.as_mut().poll_read(cx, &mut rest)?); + + tracing::trace!("read {} bytes off of payload", rest.filled().len()); + *offset += rest.filled().len(); + + if *offset == *msg_size { + tracing::trace!("read full payload (all {} bytes)", *offset); + *state = State::ReadHeader { + header_buf: Default::default(), + offset: 0, + }; + } + + rest.filled().len() + }; + + let new_filled = buf.filled().len() + just_read; + buf.set_filled(new_filled); + + return task::Poll::Ready(Ok(())); + } + State::Passthrough => { + // we encountered EOF while reading, or saw an invalid header and we're just + // passing reads through without doing any sort of processing now. + return io.poll_read(cx, buf); + } + } + } + } +} + +impl<IO> AsyncReadReady for CorkStream<IO> +where + IO: AsyncReadReady, +{ + fn poll_read_ready(&self, cx: &mut task::Context<'_>) -> task::Poll<io::Result<()>> { + self.io.poll_read_ready(cx) + } +} + +impl<IO> AsyncWrite for CorkStream<IO> +where + IO: AsyncWrite, +{ + #[inline] + fn poll_write( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + buf: &[u8], + ) -> task::Poll<io::Result<usize>> { + let io = unsafe { self.map_unchecked_mut(|s| &mut s.io) }; + io.poll_write(cx, buf) + } + + #[inline] + fn poll_flush(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> task::Poll<io::Result<()>> { + let io = unsafe { self.map_unchecked_mut(|s| &mut s.io) }; + io.poll_flush(cx) + } + + #[inline] + fn poll_shutdown( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + ) -> task::Poll<io::Result<()>> { + let io = unsafe { self.map_unchecked_mut(|s| &mut s.io) }; + io.poll_shutdown(cx) + } +} + +fn decode_header(b: [u8; 5]) -> Option<(rustls::ContentType, rustls::ProtocolVersion, u16)> { + let typ = rustls::ContentType::read_bytes(&b[0..1]).ok()?; + let version = rustls::ProtocolVersion::read_bytes(&b[1..3]).ok()?; + // this is dumb but it looks less scary than `.try_into().unwrap()`: + let len: u16 = u16::from_be_bytes([b[3], b[4]]); + Some((typ, version, len)) +} diff --git a/dstack/vendor/ktls/src/ffi.rs b/dstack/vendor/ktls/src/ffi.rs new file mode 100644 index 000000000..136553772 --- /dev/null +++ b/dstack/vendor/ktls/src/ffi.rs @@ -0,0 +1,314 @@ +use std::os::unix::prelude::RawFd; + +use ktls_sys::bindings as ktls; +use rustls::{ + internal::msgs::{enums::AlertLevel, message::Message}, + AlertDescription, ConnectionTrafficSecrets, SupportedCipherSuite, +}; + +pub(crate) const TLS_1_2_VERSION_NUMBER: u16 = (((ktls::TLS_1_2_VERSION_MAJOR & 0xFF) as u16) << 8) + | ((ktls::TLS_1_2_VERSION_MINOR & 0xFF) as u16); + +pub(crate) const TLS_1_3_VERSION_NUMBER: u16 = (((ktls::TLS_1_3_VERSION_MAJOR & 0xFF) as u16) << 8) + | ((ktls::TLS_1_3_VERSION_MINOR & 0xFF) as u16); + +/// `setsockopt` level constant: TCP +const SOL_TCP: libc::c_int = 6; + +/// `setsockopt` SOL_TCP name constant: "upper level protocol" +const TCP_ULP: libc::c_int = 31; + +/// `setsockopt` level constant: TLS +const SOL_TLS: libc::c_int = 282; + +/// `setsockopt` SOL_TLS level constant: transmit (write) +const TLS_TX: libc::c_int = 1; + +/// `setsockopt` SOL_TLS level constant: receive (read) +const TLX_RX: libc::c_int = 2; + +pub fn setup_ulp(fd: RawFd) -> std::io::Result<()> { + unsafe { + if libc::setsockopt( + fd, + SOL_TCP, + TCP_ULP, + "tls".as_ptr() as *const libc::c_void, + 3, + ) < 0 + { + return Err(std::io::Error::last_os_error()); + } + } + + Ok(()) +} + +#[derive(Clone, Copy, Debug)] +pub enum Direction { + // Transmit + Tx, + // Receive + Rx, +} + +impl From<Direction> for libc::c_int { + fn from(val: Direction) -> Self { + match val { + Direction::Tx => TLS_TX, + Direction::Rx => TLX_RX, + } + } +} + +#[allow(dead_code)] +pub enum CryptoInfo { + AesGcm128(ktls::tls12_crypto_info_aes_gcm_128), + AesGcm256(ktls::tls12_crypto_info_aes_gcm_256), + AesCcm128(ktls::tls12_crypto_info_aes_ccm_128), + Chacha20Poly1305(ktls::tls12_crypto_info_chacha20_poly1305), + Sm4Gcm(ktls::tls12_crypto_info_sm4_gcm), + Sm4Ccm(ktls::tls12_crypto_info_sm4_ccm), +} + +impl CryptoInfo { + /// Return the system struct as a pointer. + pub fn as_ptr(&self) -> *const libc::c_void { + match self { + CryptoInfo::AesGcm128(info) => info as *const _ as *const libc::c_void, + CryptoInfo::AesGcm256(info) => info as *const _ as *const libc::c_void, + CryptoInfo::AesCcm128(info) => info as *const _ as *const libc::c_void, + CryptoInfo::Chacha20Poly1305(info) => info as *const _ as *const libc::c_void, + CryptoInfo::Sm4Gcm(info) => info as *const _ as *const libc::c_void, + CryptoInfo::Sm4Ccm(info) => info as *const _ as *const libc::c_void, + } + } + + /// Return the system struct size. + pub fn size(&self) -> usize { + match self { + CryptoInfo::AesGcm128(_) => std::mem::size_of::<ktls::tls12_crypto_info_aes_gcm_128>(), + CryptoInfo::AesGcm256(_) => std::mem::size_of::<ktls::tls12_crypto_info_aes_gcm_256>(), + CryptoInfo::AesCcm128(_) => std::mem::size_of::<ktls::tls12_crypto_info_aes_ccm_128>(), + CryptoInfo::Chacha20Poly1305(_) => { + std::mem::size_of::<ktls::tls12_crypto_info_chacha20_poly1305>() + } + CryptoInfo::Sm4Gcm(_) => std::mem::size_of::<ktls::tls12_crypto_info_sm4_gcm>(), + CryptoInfo::Sm4Ccm(_) => std::mem::size_of::<ktls::tls12_crypto_info_sm4_ccm>(), + } + } +} + +#[derive(thiserror::Error, Debug)] +pub enum KtlsCompatibilityError { + #[error("cipher suite not supported with kTLS: {0:?}")] + UnsupportedCipherSuite(SupportedCipherSuite), + + #[error("wrong size key")] + WrongSizeKey, + + #[error("wrong size iv")] + WrongSizeIv, +} + +impl CryptoInfo { + /// Try to convert rustls cipher suite and secrets into a `CryptoInfo`. + pub fn from_rustls( + cipher_suite: SupportedCipherSuite, + (seq, secrets): (u64, ConnectionTrafficSecrets), + ) -> Result<CryptoInfo, KtlsCompatibilityError> { + let version = match cipher_suite { + SupportedCipherSuite::Tls12(..) => TLS_1_2_VERSION_NUMBER, + SupportedCipherSuite::Tls13(..) => TLS_1_3_VERSION_NUMBER, + }; + + Ok(match secrets { + ConnectionTrafficSecrets::Aes128Gcm { key, iv } => { + // see https://github.com/rustls/rustls/issues/1833, between + // rustls 0.21 and 0.22, the extract_keys codepath was changed, + // so, for TLS 1.2, both GCM-128 and GCM-256 return the + // Aes128Gcm variant. + + match key.as_ref().len() { + 16 => CryptoInfo::AesGcm128(ktls::tls12_crypto_info_aes_gcm_128 { + info: ktls::tls_crypto_info { + version, + cipher_type: ktls::TLS_CIPHER_AES_GCM_128 as _, + }, + iv: iv + .as_ref() + .get(4..) + .expect("AES-GCM-128 iv is 8 bytes") + .try_into() + .expect("AES-GCM-128 iv is 8 bytes"), + key: key + .as_ref() + .try_into() + .expect("AES-GCM-128 key is 16 bytes"), + salt: iv + .as_ref() + .get(..4) + .expect("AES-GCM-128 salt is 4 bytes") + .try_into() + .expect("AES-GCM-128 salt is 4 bytes"), + rec_seq: seq.to_be_bytes(), + }), + 32 => CryptoInfo::AesGcm256(ktls::tls12_crypto_info_aes_gcm_256 { + info: ktls::tls_crypto_info { + version, + cipher_type: ktls::TLS_CIPHER_AES_GCM_256 as _, + }, + iv: iv + .as_ref() + .get(4..) + .expect("AES-GCM-256 iv is 8 bytes") + .try_into() + .expect("AES-GCM-256 iv is 8 bytes"), + key: key + .as_ref() + .try_into() + .expect("AES-GCM-256 key is 32 bytes"), + salt: iv + .as_ref() + .get(..4) + .expect("AES-GCM-256 salt is 4 bytes") + .try_into() + .expect("AES-GCM-256 salt is 4 bytes"), + rec_seq: seq.to_be_bytes(), + }), + _ => unreachable!("GCM key length is not 16 or 32"), + } + } + ConnectionTrafficSecrets::Aes256Gcm { key, iv } => { + CryptoInfo::AesGcm256(ktls::tls12_crypto_info_aes_gcm_256 { + info: ktls::tls_crypto_info { + version, + cipher_type: ktls::TLS_CIPHER_AES_GCM_256 as _, + }, + iv: iv + .as_ref() + .get(4..) + .expect("AES-GCM-256 iv is 8 bytes") + .try_into() + .expect("AES-GCM-256 iv is 8 bytes"), + key: key + .as_ref() + .try_into() + .expect("AES-GCM-256 key is 32 bytes"), + salt: iv + .as_ref() + .get(..4) + .expect("AES-GCM-256 salt is 4 bytes") + .try_into() + .expect("AES-GCM-256 salt is 4 bytes"), + rec_seq: seq.to_be_bytes(), + }) + } + ConnectionTrafficSecrets::Chacha20Poly1305 { key, iv } => { + CryptoInfo::Chacha20Poly1305(ktls::tls12_crypto_info_chacha20_poly1305 { + info: ktls::tls_crypto_info { + version, + cipher_type: ktls::TLS_CIPHER_CHACHA20_POLY1305 as _, + }, + iv: iv + .as_ref() + .try_into() + .expect("Chacha20-Poly1305 iv is 12 bytes"), + key: key + .as_ref() + .try_into() + .expect("Chacha20-Poly1305 key is 32 bytes"), + salt: ktls::__IncompleteArrayField::new(), + rec_seq: seq.to_be_bytes(), + }) + } + _ => { + return Err(KtlsCompatibilityError::UnsupportedCipherSuite(cipher_suite)); + } + }) + } +} + +pub fn setup_tls_info(fd: RawFd, dir: Direction, info: CryptoInfo) -> Result<(), crate::Error> { + let ret = unsafe { libc::setsockopt(fd, SOL_TLS, dir.into(), info.as_ptr(), info.size() as _) }; + if ret < 0 { + return Err(crate::Error::TlsCryptoInfoError( + std::io::Error::last_os_error(), + )); + } + Ok(()) +} + +const TLS_SET_RECORD_TYPE: libc::c_int = 1; +const ALERT: u8 = 0x15; + +// Yes, really. cmsg components are aligned to [libc::c_long] +#[cfg_attr(target_pointer_width = "32", repr(C, align(4)))] +#[cfg_attr(target_pointer_width = "64", repr(C, align(8)))] +struct Cmsg<const N: usize> { + hdr: libc::cmsghdr, + data: [u8; N], +} + +impl<const N: usize> Cmsg<N> { + fn new(level: i32, typ: i32, data: [u8; N]) -> Self { + // dstack patch (rustls/ktls#70): built field by field from a zeroed + // value rather than with struct literal syntax. musl's `cmsghdr` has a + // real `__pad1` member that libc keeps private, so the literal cannot + // name every field and does not compile for musl targets. + // + // SAFETY: `cmsghdr` is a plain C struct with no invalid bit patterns, + // so an all-zero value is valid, and every meaningful field is assigned + // immediately below. + let hdr = unsafe { + let mut hdr: libc::cmsghdr = std::mem::zeroed(); + // on Linux this is a usize, on macOS this is a u32 + #[allow(clippy::unnecessary_cast)] + { + hdr.cmsg_len = (memoffset::offset_of!(Self, data) + N) as _; + } + hdr.cmsg_level = level; + hdr.cmsg_type = typ; + hdr + }; + Self { hdr, data } + } +} + +pub fn send_close_notify(fd: RawFd) -> std::io::Result<()> { + let mut data = vec![]; + Message::build_alert(AlertLevel::Warning, AlertDescription::CloseNotify) + .payload + .encode(&mut data); + + let mut cmsg = Cmsg::new(SOL_TLS, TLS_SET_RECORD_TYPE, [ALERT]); + + // dstack patch (rustls/ktls#70): same reason as `Cmsg::new` above -- musl's + // `msghdr` carries private `__pad1`/`__pad2` members, so struct literal + // syntax is rejected outright there. Hoisting `iov` into a named local is + // also what upstream's patch does, and it is what makes the pointer stored + // in `msg_iov` outlive the `sendmsg` call rather than a temporary that ends + // with its own statement. + let mut iov = libc::iovec { + iov_base: data.as_mut_ptr() as _, + iov_len: data.len(), + }; + // SAFETY: as above -- `msghdr` is a plain C struct, an all-zero value is + // valid, and the fields this call depends on are assigned right after. + // `msg_name`/`msg_namelen`/`msg_flags` are left zeroed, which is what the + // struct literal set them to. + let msg = unsafe { + let mut msg: libc::msghdr = std::mem::zeroed(); + msg.msg_iov = &mut iov; + msg.msg_iovlen = 1; + msg.msg_control = &mut cmsg as *mut _ as *mut _; + msg.msg_controllen = cmsg.hdr.cmsg_len as _; + msg + }; + + let ret = unsafe { libc::sendmsg(fd, &msg, 0) }; + if ret < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} diff --git a/dstack/vendor/ktls/src/ktls_stream.rs b/dstack/vendor/ktls/src/ktls_stream.rs new file mode 100644 index 000000000..5ddf0c53f --- /dev/null +++ b/dstack/vendor/ktls/src/ktls_stream.rs @@ -0,0 +1,308 @@ +use nix::{ + errno::Errno, + sys::socket::{recvmsg, ControlMessageOwned, MsgFlags, SockaddrIn, TlsGetRecordType}, +}; +use num_enum::FromPrimitive; +use std::{ + io::{self, IoSliceMut}, + os::unix::prelude::AsRawFd, + pin::Pin, + task, +}; + +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +use crate::AsyncReadReady; + +// A wrapper around `IO` that sends a `close_notify` when shut down or dropped. +pin_project_lite::pin_project! { + pub struct KtlsStream<IO> + where + IO: AsRawFd + { + #[pin] + inner: IO, + write_closed: bool, + read_closed: bool, + drained: Option<(usize, Vec<u8>)>, + } +} + +impl<IO> KtlsStream<IO> +where + IO: AsRawFd, +{ + pub fn new(inner: IO, drained: Option<Vec<u8>>) -> Self { + Self { + inner, + write_closed: false, + read_closed: false, + drained: drained.map(|drained| (0, drained)), + } + } + + /// Return the drained data + the original I/O + pub fn into_raw(self) -> (Option<Vec<u8>>, IO) { + (self.drained.map(|(_, drained)| drained), self.inner) + } + + /// Returns a reference to the original I/O + pub fn get_ref(&self) -> &IO { + &self.inner + } + + /// Returns a mut reference to the original I/O + pub fn get_mut(&mut self) -> &mut IO { + &mut self.inner + } +} + +#[derive(Debug, PartialEq, Clone, Copy, num_enum::FromPrimitive)] +#[repr(u8)] +enum TlsAlertLevel { + Warning = 1, + Fatal = 2, + #[num_enum(catch_all)] + Other(u8), +} + +#[derive(Debug, PartialEq, Clone, Copy, num_enum::FromPrimitive)] +#[repr(u8)] +enum TlsAlertDescription { + CloseNotify = 0, + #[num_enum(catch_all)] + Other(u8), +} + +impl<IO> AsyncRead for KtlsStream<IO> +where + IO: AsRawFd + AsyncRead + AsyncReadReady, +{ + fn poll_read( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> task::Poll<io::Result<()>> { + tracing::trace!(buf.remaining = %buf.remaining(), "KtlsStream::poll_read"); + + if self.read_closed { + return task::Poll::Ready(Ok(())); + } + + if buf.remaining() == 0 { + return task::Poll::Ready(Ok(())); + } + + let mut this = self.project(); + + if let Some((drain_index, drained)) = this.drained.as_mut() { + let drained = &drained[*drain_index..]; + let len = std::cmp::min(buf.remaining(), drained.len()); + + tracing::trace!(%len, "KtlsStream::poll_read, can take from drain"); + buf.put_slice(&drained[..len]); + + *drain_index += len; + if *drain_index >= drained.len() { + tracing::trace!("KtlsStream::poll_read, done draining"); + *this.drained = None; + } + cx.waker().wake_by_ref(); + + tracing::trace!("KtlsStream::poll_read, returning after drain"); + return task::Poll::Ready(Ok(())); + } + + let read_res = this.inner.as_mut().poll_read(cx, buf); + if let task::Poll::Ready(Err(e)) = &read_res { + // 5 is a generic "input/output error", it happens when + // using poll_read on a kTLS socket that just received + // a control message + if let Some(5) = e.raw_os_error() { + // could be a control message, let's check + let fd = this.inner.as_raw_fd(); + + // XXX: recvmsg wants a `&mut Vec<u8>` so it's able to resize it + // I guess? Or so there's a clear separation between uninitialized + // and initialized? We could probably get read of that heap alloc, idk. + + // let mut cmsgspace = + // [0u8; unsafe { libc::CMSG_SPACE(std::mem::size_of::<u8>() as _) as _ }]; + let mut cmsgspace = Vec::with_capacity(unsafe { + libc::CMSG_SPACE(std::mem::size_of::<u8>() as _) as _ + }); + + let mut iov = [IoSliceMut::new(buf.initialize_unfilled())]; + let flags = MsgFlags::empty(); + + let r = recvmsg::<SockaddrIn>(fd, &mut iov, Some(&mut cmsgspace), flags); + let r = match r { + Ok(r) => r, + Err(Errno::EAGAIN) => { + unreachable!("expected a control message, got EAGAIN") + } + Err(e) => { + // ok I guess it really failed then + tracing::trace!(?e, "recvmsg failed"); + return Err(e.into()).into(); + } + }; + let cmsg = r + .cmsgs()? + .next() + .expect("we should've received exactly one control message"); + + let record_type = match cmsg { + ControlMessageOwned::TlsGetRecordType(t) => t, + _ => panic!("unexpected cmsg type: {cmsg:#?}"), + }; + + match record_type { + TlsGetRecordType::ChangeCipherSpec => { + panic!("change_cipher_spec isn't supported by the ktls crate") + } + TlsGetRecordType::Alert => { + // the alert level and description are in iovs + let iov = r.iovs().next().expect("expected data in iovs"); + + let (level, description) = match iov { + [] => { + // we have an early return case for that + unreachable!(); + } + &[level] => { + // https://github.com/facebookincubator/fizz/blob/fff6d9d49d3c554ab66b58822d1e1fe93e8d80f2/fizz/experimental/ktls/AsyncKTLSSocket.cpp#L144 + // + // Since all alerts (even warning-level alerts) + // signal the abort of a TLS session, we do not + // need to worry about additional application + // data. + // + // If we only have half the alert (because the + // user passed a buffer of size 1), just assume + // it's a close_notify + ( + TlsAlertLevel::from_primitive(level), + TlsAlertDescription::CloseNotify, + ) + } + &[level, description] => ( + TlsAlertLevel::from_primitive(level), + TlsAlertDescription::from_primitive(description), + ), + _ => { + unreachable!( + "TLS alerts are exactly 2 bytes, your kTLS is misbehaving" + ); + } + }; + + match (level, description) { + // https://datatracker.ietf.org/doc/html/rfc5246#section-7.2 + // alerts we should handle are ones with fatal level or a + // close_notify + (_, TlsAlertDescription::CloseNotify) | (TlsAlertLevel::Fatal, _) => { + tracing::trace!(?level, ?description, "got TLS alert"); + *this.read_closed = true; + *this.write_closed = true; + if let Err(e) = + crate::ffi::send_close_notify(this.inner.as_raw_fd()) + { + return Err(e).into(); + } + // the file descriptor will be closed when the stream is dropped, + // we already protect against writes-after-close_notify through + // the write_closed flag + return task::Poll::Ready(Ok(())); + } + _ => { + // we got something we probably can't handle + } + } + return task::Poll::Ready(Ok(())); + } + TlsGetRecordType::Handshake => { + // TODO: this is where we receive TLS 1.3 resumption tickets, + // should those be stored anywhere? I'm not even sure what + // format they have at this point + tracing::trace!( + "ignoring handshake message (probably a resumption ticket)" + ); + } + TlsGetRecordType::ApplicationData => { + unreachable!("received TLS application in recvmsg, this is supposed to happen in the poll_read codepath") + } + TlsGetRecordType::Unknown(t) => { + // just ignore the record? + tracing::trace!("received record_type {t:#?}"); + } + _ => { + tracing::trace!("received unsupported record type"); + } + }; + + // FIXME: this is hacky, but can we do better? + // after we handled (..ignored) the control message, we don't + // know whether the socket is still ready to be read or not. + // + // we could try looping (tricky code structure), but we can't, + // for example, just call `poll_read`, which might fail not + // not with EAGAIN/EWOULDBLOCK, but because _another_ control + // message is available. + cx.waker().wake_by_ref(); + return task::Poll::Pending; + } + } + + read_res + } +} + +impl<IO> AsyncWrite for KtlsStream<IO> +where + IO: AsRawFd + AsyncWrite, +{ + fn poll_write( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + buf: &[u8], + ) -> task::Poll<io::Result<usize>> { + if self.write_closed { + return task::Poll::Ready(Ok(0)); + } + + self.project().inner.poll_write(cx, buf) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> task::Poll<io::Result<()>> { + self.project().inner.poll_flush(cx) + } + + fn poll_shutdown( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + ) -> task::Poll<io::Result<()>> { + let this = self.project(); + + if !*this.write_closed { + // they didn't hang up on us, we're nicely being asked to shut down, + // let's send a close_notify (and not wait for them to send it back) + *this.write_closed = true; + if let Err(e) = crate::ffi::send_close_notify(this.inner.as_raw_fd()) { + return Err(e).into(); + } + } + + // this ends up closing the inner file descriptor no matter what + this.inner.poll_shutdown(cx) + } +} + +impl<IO> AsRawFd for KtlsStream<IO> +where + IO: AsRawFd, +{ + fn as_raw_fd(&self) -> std::os::unix::prelude::RawFd { + self.inner.as_raw_fd() + } +} diff --git a/dstack/vendor/ktls/src/lib.rs b/dstack/vendor/ktls/src/lib.rs new file mode 100644 index 000000000..774980b5d --- /dev/null +++ b/dstack/vendor/ktls/src/lib.rs @@ -0,0 +1,463 @@ +// dstack patch: vendored third-party code is carried to be patched as little as +// possible, not rewritten to this workspace's lint policy. A path dependency is +// a local package, so cargo does not cap its lints the way it would for a +// crates.io one, and CI's `-D clippy::expect_used -D clippy::unwrap_used` would +// otherwise fail on upstream code unrelated to the musl fix. Attribute levels +// beat command-line ones, which a `[lints]` table in Cargo.toml does not. +#![allow(clippy::expect_used, clippy::unwrap_used)] + +use ffi::{setup_tls_info, setup_ulp, KtlsCompatibilityError}; +use futures_util::future::try_join_all; +use ktls_sys::bindings as sys; +use rustls::{Connection, SupportedCipherSuite, SupportedProtocolVersion}; + +#[cfg(all(not(feature = "ring"), not(feature = "aws_lc_rs")))] +compile_error!("This crate needs wither the 'ring' or 'aws_lc_rs' feature enabled"); +#[cfg(all(feature = "ring", feature = "aws_lc_rs"))] +compile_error!("The 'ring' and 'aws_lc_rs' features are mutually exclusive"); +#[cfg(feature = "aws_lc_rs")] +use rustls::crypto::aws_lc_rs::cipher_suite; +#[cfg(feature = "ring")] +use rustls::crypto::ring::cipher_suite; + +use smallvec::SmallVec; +use std::{ + future::Future, + io, + net::SocketAddr, + os::unix::prelude::{AsRawFd, RawFd}, +}; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWrite}, + net::{TcpListener, TcpStream}, +}; + +mod ffi; +pub use crate::ffi::CryptoInfo; +// dstack patch: the offload path splices the bare descriptor rather than going +// through `KtlsStream`, so it has to send the TLS close itself; upstream only +// calls this from `KtlsStream::poll_shutdown` and keeps it private. +pub use crate::ffi::send_close_notify; + +mod async_read_ready; +pub use async_read_ready::AsyncReadReady; + +mod ktls_stream; +pub use ktls_stream::KtlsStream; + +mod cork_stream; +pub use cork_stream::CorkStream; + +#[derive(Debug, Default)] +pub struct CompatibleCiphers { + pub tls12: CompatibleCiphersForVersion, + pub tls13: CompatibleCiphersForVersion, +} + +#[derive(Debug, Default)] +pub struct CompatibleCiphersForVersion { + pub aes_gcm_128: bool, + pub aes_gcm_256: bool, + pub chacha20_poly1305: bool, +} + +impl CompatibleCiphers { + /// List compatible ciphers. This listens on a TCP socket and blocks for a + /// little while. Do once at the very start of a program. Should probably be + /// behind a lazy_static / once_cell + pub async fn new() -> io::Result<Self> { + let mut ciphers = CompatibleCiphers::default(); + + let ln = TcpListener::bind("0.0.0.0:0").await?; + let local_addr = ln.local_addr()?; + + // Accepted conns of ln + let mut accepted_conns: SmallVec<[TcpStream; 12]> = SmallVec::new(); + + let accept_conns_fut = async { + loop { + if let Ok((conn, _addr)) = ln.accept().await { + accepted_conns.push(conn); + } + } + }; + + ciphers.test_ciphers(local_addr, accept_conns_fut).await?; + + Ok(ciphers) + } + + async fn test_ciphers( + &mut self, + local_addr: SocketAddr, + accept_conns_fut: impl Future<Output = ()>, + ) -> io::Result<()> { + let ciphers: Vec<(SupportedCipherSuite, &mut bool)> = vec![ + ( + cipher_suite::TLS13_AES_128_GCM_SHA256, + &mut self.tls13.aes_gcm_128, + ), + ( + cipher_suite::TLS13_AES_256_GCM_SHA384, + &mut self.tls13.aes_gcm_256, + ), + ( + cipher_suite::TLS13_CHACHA20_POLY1305_SHA256, + &mut self.tls13.chacha20_poly1305, + ), + ( + cipher_suite::TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + &mut self.tls12.aes_gcm_128, + ), + ( + cipher_suite::TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + &mut self.tls12.aes_gcm_256, + ), + ( + cipher_suite::TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + &mut self.tls12.chacha20_poly1305, + ), + ]; + + let create_connections_fut = + try_join_all((0..ciphers.len()).map(|_| TcpStream::connect(local_addr))); + + let socks = tokio::select! { + // Use biased here to optimize performance. + // + // With biased, tokio::select! would first poll create_connections_fut, + // which would poll all `TcpStream::connect` futures and requests + // new connections to `ln` then returns `Poll::Pending`. + // + // Then accept_conns_fut would be polled, which accepts all pending + // connections, wake up create_connections_fut then returns + // `Poll::Pending`. + // + // Finally, create_connections_fut wakes up and all connections + // are ready, the result is collected into a Vec and ends + // the tokio::select!. + biased; + + res = create_connections_fut => res?, + _ = accept_conns_fut => unreachable!(), + }; + + assert_eq!(ciphers.len(), socks.len()); + + ciphers + .into_iter() + .zip(socks) + .for_each(|((cipher_suite, field), sock)| { + *field = sample_cipher_setup(&sock, cipher_suite).is_ok(); + }); + + Ok(()) + } + + /// Returns true if we're reasonably confident that functions like + /// [config_ktls_client] and [config_ktls_server] will succeed. + pub fn is_compatible(&self, suite: SupportedCipherSuite) -> bool { + let kcs = match KtlsCipherSuite::try_from(suite) { + Ok(kcs) => kcs, + Err(_) => return false, + }; + + let fields = match kcs.version { + KtlsVersion::TLS12 => &self.tls12, + KtlsVersion::TLS13 => &self.tls13, + }; + + match kcs.typ { + KtlsCipherType::AesGcm128 => fields.aes_gcm_128, + KtlsCipherType::AesGcm256 => fields.aes_gcm_256, + KtlsCipherType::Chacha20Poly1305 => fields.chacha20_poly1305, + } + } +} + +fn sample_cipher_setup(sock: &TcpStream, cipher_suite: SupportedCipherSuite) -> Result<(), Error> { + let kcs = match KtlsCipherSuite::try_from(cipher_suite) { + Ok(kcs) => kcs, + Err(_) => panic!("unsupported cipher suite"), + }; + + let ffi_version = match kcs.version { + KtlsVersion::TLS12 => ffi::TLS_1_2_VERSION_NUMBER, + KtlsVersion::TLS13 => ffi::TLS_1_3_VERSION_NUMBER, + }; + + let crypto_info = match kcs.typ { + KtlsCipherType::AesGcm128 => CryptoInfo::AesGcm128(sys::tls12_crypto_info_aes_gcm_128 { + info: sys::tls_crypto_info { + version: ffi_version, + cipher_type: sys::TLS_CIPHER_AES_GCM_128 as _, + }, + iv: Default::default(), + key: Default::default(), + salt: Default::default(), + rec_seq: Default::default(), + }), + KtlsCipherType::AesGcm256 => CryptoInfo::AesGcm256(sys::tls12_crypto_info_aes_gcm_256 { + info: sys::tls_crypto_info { + version: ffi_version, + cipher_type: sys::TLS_CIPHER_AES_GCM_256 as _, + }, + iv: Default::default(), + key: Default::default(), + salt: Default::default(), + rec_seq: Default::default(), + }), + KtlsCipherType::Chacha20Poly1305 => { + CryptoInfo::Chacha20Poly1305(sys::tls12_crypto_info_chacha20_poly1305 { + info: sys::tls_crypto_info { + version: ffi_version, + cipher_type: sys::TLS_CIPHER_CHACHA20_POLY1305 as _, + }, + iv: Default::default(), + key: Default::default(), + salt: Default::default(), + rec_seq: Default::default(), + }) + } + }; + let fd = sock.as_raw_fd(); + + setup_ulp(fd).map_err(Error::UlpError)?; + + setup_tls_info(fd, ffi::Direction::Tx, crypto_info)?; + + Ok(()) +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("failed to enable TLS ULP (upper level protocol): {0}")] + UlpError(#[source] std::io::Error), + + #[error("kTLS compatibility error: {0}")] + KtlsCompatibility(#[from] KtlsCompatibilityError), + + #[error("failed to export secrets")] + ExportSecrets(#[source] rustls::Error), + + #[error("failed to configure tx/rx (unsupported cipher?): {0}")] + TlsCryptoInfoError(#[source] std::io::Error), + + #[error("an I/O occured while draining the rustls stream: {0}")] + DrainError(#[source] std::io::Error), + + #[error("no negotiated cipher suite: call config_ktls_* only /after/ the handshake")] + NoNegotiatedCipherSuite, +} + +/// Configure kTLS for this socket. If this call succeeds, data can be written +/// and read from this socket, and the kernel takes care of encryption +/// transparently. I'm not clear how rekeying is handled (probably via control +/// messages, but can't find a code sample for it). +/// +/// The inner IO type must be wrapped in [CorkStream] since it's the only way +/// to drain a rustls stream cleanly. See its documentation for details. +pub async fn config_ktls_server<IO>( + mut stream: tokio_rustls::server::TlsStream<CorkStream<IO>>, +) -> Result<KtlsStream<IO>, Error> +where + IO: AsRawFd + AsyncRead + AsyncReadReady + AsyncWrite + Unpin, +{ + stream.get_mut().0.corked = true; + let drained = drain(&mut stream).await.map_err(Error::DrainError)?; + let (io, conn) = stream.into_inner(); + let io = io.io; + + setup_inner(io.as_raw_fd(), Connection::Server(conn))?; + Ok(KtlsStream::new(io, drained)) +} + +/// Configure kTLS for this socket. If this call succeeds, data can be +/// written and read from this socket, and the kernel takes care of encryption +/// (and key updates, etc.) transparently. +/// +/// The inner IO type must be wrapped in [CorkStream] since it's the only way +/// to drain a rustls stream cleanly. See its documentation for details. +pub async fn config_ktls_client<IO>( + mut stream: tokio_rustls::client::TlsStream<CorkStream<IO>>, +) -> Result<KtlsStream<IO>, Error> +where + IO: AsRawFd + AsyncRead + AsyncWrite + Unpin, +{ + stream.get_mut().0.corked = true; + let drained = drain(&mut stream).await.map_err(Error::DrainError)?; + let (io, conn) = stream.into_inner(); + let io = io.io; + + setup_inner(io.as_raw_fd(), Connection::Client(conn))?; + Ok(KtlsStream::new(io, drained)) +} + +/// Read all the bytes we can read without blocking. This is used to drained the +/// already-decrypted buffer from a tokio-rustls I/O type +async fn drain(stream: &mut (impl AsyncRead + Unpin)) -> std::io::Result<Option<Vec<u8>>> { + tracing::trace!("Draining rustls stream"); + let mut drained = vec![0u8; 128 * 1024]; + let mut filled = 0; + + loop { + tracing::trace!("stream.read called"); + let n = match stream.read(&mut drained[filled..]).await { + Ok(n) => n, + Err(ref e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { + // actually this is expected for us! + tracing::trace!("stream.read returned UnexpectedEof, that's expected for us"); + break; + } + Err(e) => { + tracing::trace!("stream.read returned error: {e}"); + return Err(e); + } + }; + tracing::trace!("stream.read returned {n}"); + if n == 0 { + // that's what CorkStream returns when it's at a message boundary + break; + } + filled += n; + } + + let maybe_drained = if filled == 0 { + None + } else { + tracing::trace!("Draining rustls stream done: drained {filled} bytes"); + drained.resize(filled, 0); + Some(drained) + }; + Ok(maybe_drained) +} + +fn setup_inner(fd: RawFd, conn: Connection) -> Result<(), Error> { + let cipher_suite = match conn.negotiated_cipher_suite() { + Some(cipher_suite) => cipher_suite, + None => { + return Err(Error::NoNegotiatedCipherSuite); + } + }; + + let secrets = match conn.dangerous_extract_secrets() { + Ok(secrets) => secrets, + Err(err) => return Err(Error::ExportSecrets(err)), + }; + + ffi::setup_ulp(fd).map_err(Error::UlpError)?; + + let tx = CryptoInfo::from_rustls(cipher_suite, secrets.tx)?; + setup_tls_info(fd, ffi::Direction::Tx, tx)?; + + let rx = CryptoInfo::from_rustls(cipher_suite, secrets.rx)?; + setup_tls_info(fd, ffi::Direction::Rx, rx)?; + + Ok(()) +} + +/// TLS versions supported by this crate +#[non_exhaustive] +#[derive(Debug, Clone, Copy)] +pub enum KtlsVersion { + TLS12, + TLS13, +} + +impl KtlsVersion { + /// Converts into the equivalent rustls [SupportedProtocolVersion] + pub fn as_supported_version(&self) -> &'static SupportedProtocolVersion { + match self { + KtlsVersion::TLS12 => &rustls::version::TLS12, + KtlsVersion::TLS13 => &rustls::version::TLS13, + } + } +} + +/// A TLS cipher suite. Used mostly internally. +#[derive(Clone, Copy)] +pub struct KtlsCipherSuite { + /// The TLS version + pub version: KtlsVersion, + + /// The cipher type + pub typ: KtlsCipherType, +} + +/// Cipher types supported by this crate +#[non_exhaustive] +#[derive(Debug, Clone, Copy)] +pub enum KtlsCipherType { + AesGcm128, + AesGcm256, + Chacha20Poly1305, +} + +#[derive(Debug, thiserror::Error)] +pub enum CipherSuiteError { + #[error("TLS 1.2 support not built in")] + Tls12NotBuiltIn, + + #[error("unsupported cipher suite")] + UnsupportedCipherSuite(SupportedCipherSuite), +} + +impl TryFrom<SupportedCipherSuite> for KtlsCipherSuite { + type Error = CipherSuiteError; + + fn try_from(#[allow(unused)] suite: SupportedCipherSuite) -> Result<Self, Self::Error> { + { + let version = match suite { + SupportedCipherSuite::Tls12(..) => { + if !cfg!(feature = "tls12") { + return Err(CipherSuiteError::Tls12NotBuiltIn); + } + KtlsVersion::TLS12 + } + SupportedCipherSuite::Tls13(..) => KtlsVersion::TLS13, + }; + + let family = { + if suite == cipher_suite::TLS13_AES_128_GCM_SHA256 { + KtlsCipherType::AesGcm128 + } else if suite == cipher_suite::TLS13_AES_256_GCM_SHA384 { + KtlsCipherType::AesGcm256 + } else if suite == cipher_suite::TLS13_CHACHA20_POLY1305_SHA256 { + KtlsCipherType::Chacha20Poly1305 + } else if suite == cipher_suite::TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 { + KtlsCipherType::AesGcm128 + } else if suite == cipher_suite::TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 { + KtlsCipherType::AesGcm256 + } else if suite == cipher_suite::TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 { + KtlsCipherType::Chacha20Poly1305 + } else { + return Err(CipherSuiteError::UnsupportedCipherSuite(suite)); + } + }; + + Ok(Self { + typ: family, + version, + }) + } + } +} + +impl KtlsCipherSuite { + pub fn as_supported_cipher_suite(&self) -> SupportedCipherSuite { + match self.version { + KtlsVersion::TLS12 => match self.typ { + KtlsCipherType::AesGcm128 => cipher_suite::TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + KtlsCipherType::AesGcm256 => cipher_suite::TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + KtlsCipherType::Chacha20Poly1305 => { + cipher_suite::TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 + } + }, + KtlsVersion::TLS13 => match self.typ { + KtlsCipherType::AesGcm128 => cipher_suite::TLS13_AES_128_GCM_SHA256, + KtlsCipherType::AesGcm256 => cipher_suite::TLS13_AES_256_GCM_SHA384, + KtlsCipherType::Chacha20Poly1305 => cipher_suite::TLS13_CHACHA20_POLY1305_SHA256, + }, + } + } +} diff --git a/dstack/verifier/Cargo.toml b/dstack/verifier/Cargo.toml index e48d2e10a..b9bdad654 100644 --- a/dstack/verifier/Cargo.toml +++ b/dstack/verifier/Cargo.toml @@ -34,9 +34,12 @@ tracing.workspace = true tracing-subscriber = { workspace = true, optional = true } reqwest.workspace = true tempfile.workspace = true +flate2.workspace = true +tar.workspace = true # Internal dependencies ra-tls.workspace = true +x509-parser = { workspace = true, features = ["verify"] } dstack-attest.workspace = true dstack-types.workspace = true dstack-mr.workspace = true @@ -55,3 +58,6 @@ hex-literal.workspace = true [features] default = ["binary"] binary = ["dep:clap", "dep:figment", "dep:rocket", "dep:tracing-subscriber"] + +[dev-dependencies] +rcgen.workspace = true diff --git a/dstack/verifier/README.md b/dstack/verifier/README.md index 0dada3cfe..232ede636 100644 --- a/dstack/verifier/README.md +++ b/dstack/verifier/README.md @@ -6,7 +6,7 @@ A HTTP server that provides dstack quote verification services using the same ve ### POST /verify -Verifies a dstack attestation or quote with the provided data and VM configuration. The body can be grabbed via [getQuote](https://github.com/Dstack-TEE/dstack/blob/master/sdk/curl/api.md#3-get-quote) or [attest](https://github.com/Dstack-TEE/dstack/blob/master/sdk/curl/api.md#8-attest). +Verifies a dstack attestation or quote with the provided data and VM configuration. The body can be grabbed via [getQuote](https://github.com/Dstack-TEE/dstack/blob/next/sdk/curl/api.md#3-get-quote) (Intel TDX only, and not the full evidence on GCP) or [attest](https://github.com/Dstack-TEE/dstack/blob/next/sdk/curl/api.md#7-attest) (any platform). **Request Body:** Provide either `attestation` or (`quote` + `event_log` + `vm_config`). @@ -40,7 +40,7 @@ against the returned evidence. "acpi_tables_verified": true, // true only when TDX ACPI table contents are verified "os_image_is_dev": false, // true=dev image, false=prod, null=unknown/N/A "os_image_version": "0.5.10", // dstack OS version, null if unknown - "attestation_mode": "dstack-tdx", // dstack-tdx | dstack-gcp-tdx | dstack-nitro-enclave | dstack-amd-sev-snp | dstack-aws-nitro-tpm + "tee_variant": "dstack-tdx", // dstack-tdx | dstack-gcp-tdx | dstack-nitro-enclave | dstack-amd-sev-snp | dstack-aws-nitro-tpm "report_data": "hex-encoded-64-byte-report-data", "tcb_status": "UpToDate", "advisory_ids": [], @@ -61,7 +61,7 @@ against the returned evidence. "key_provider_info": "hex-string" }, "boot_info": { - "attestationMode": "dstack-aws-nitro-tpm", + "teeVariant": "dstack-aws-nitro-tpm", "mrAggregated": "hex-string", "osImageHash": "hex-string", "mrSystem": "hex-string", @@ -100,7 +100,8 @@ You usually don't need to edit the config file. Just using the default is fine, - `image_cache_dir`: Directory for cached OS images (default: "/tmp/dstack-verifier/cache") - `image_download_url`: URL template for downloading OS images (default: dstack official releases URL) - `image_download_timeout_secs`: Download timeout in seconds (default: 300) -- `pccs_url`: PCCS URL for quote verification (default: uses Intel's public PCCS) +- `attestation.urls.pccs`: PCCS URL (default: production PCCS) +- `attestation.urls.amd_kds`: AMD KDS URL (default: AMD production KDS) ### Example Configuration File @@ -110,7 +111,9 @@ port = 8080 image_cache_dir = "/tmp/dstack-verifier/cache" image_download_url = "https://download.dstack.org/os-images/mr_{OS_IMAGE_HASH}.tar.gz" image_download_timeout_secs = 300 -# pccs_url = "https://pccs.phala.network" +[attestation.urls] +# pccs = "https://pccs.phala.network" +# amd_kds = "https://kdsintf.amd.com/vcek/v1" ``` ## Usage @@ -158,7 +161,7 @@ The verification checks that: Clients, gateways, or release validators should combine this certificate-level check with the policy fields emitted by `/verify`: accepted OS image, -`attestationMode`, `mrAggregated`, app compose hash, KMS identity, a +`teeVariant`, `mrAggregated`, app compose hash, KMS identity, a `report_data` challenge, and any deployment-specific endpoint allowlist. ### Running with Docker Compose @@ -250,19 +253,45 @@ The verifier performs the following verification steps: result as `app_info.os_image_hash_verified` (self-contained for all platforms except the TDX legacy full-image path, which reports `false`). -`details.acpi_tables_verified` is `true` only for the full-image TDX path, where the verifier recomputes ACPI table contents and checks the resulting RTMRs against the quote. It is `false` for TDX lite, which uses the quote's named ACPI DATA digests without validating table contents, and for non-TDX platforms where ACPI table verification is not applicable. +`details.acpi_tables_verified` is `true` for both TDX paths, which recompute the ACPI table contents and check them against the quote. It is `false` only for non-TDX platforms, where ACPI table verification is not applicable. All verification steps must pass for the verification to be considered valid. +### TDX ACPI table verification + +RTMR0 covers three ACPI blobs QEMU hands to OVMF (`acpi-loader`, `acpi-rsdp`, +`acpi-tables`); `acpi-tables` carries the DSDT, which is AML the guest kernel +executes. Both TDX paths regenerate those blobs from the VM shape declared in +`vm_config` (vCPU count, RAM size, PCI topology, QEMU version) and require the +recomputed digests to equal the ones the quote's event log reports, before +rebuilding the expected RTMR0 from the recomputed values. + +Verification fails closed in both directions: + +- **digest mismatch** — the tables are not the ones this VM shape produces. +- **cannot generate** — the shape is one the ACPI generator does not model + (`swtpm = true`, or a QEMU older than 8.0). There is nothing to compare + against, so the attestation is rejected rather than accepted as unverified. + +The second case is deliberate. `swtpm` and `qemu_version` are host-declared +fields that no other measurement independently constrains, so accepting an +unverifiable shape would let a host opt out of the check by declaring one. +CVMs using the TPM key provider (`swtpm = true`) therefore cannot be verified +on either TDX path, which is the pre-existing behavior of the full-image path. + +QEMU versions newer than the newest ACPI profile the verifier models are +generated with that profile, so a QEMU upgrade that leaves the ACPI ABI alone +keeps verifying; one that changes it surfaces as a digest mismatch. + ### Identifying the deployment Beyond pass/fail, the result carries a few descriptive fields so a relying party can apply its own policy: - **`os_image_is_dev`** — `true` for a development OS image, `false` for production. Dev images are built for local testing and are not hardened for production use, so a relying party generally wants to reject them. - **`os_image_version`** — the dstack OS version (e.g. `0.5.10`), useful for enforcing a minimum version. -- **`attestation_mode`** — the attestation mode that produced the verified quote, serialized as `AttestationMode`: `dstack-tdx`, `dstack-gcp-tdx`, `dstack-nitro-enclave`, `dstack-amd-sev-snp`, or `dstack-aws-nitro-tpm`. +- **`tee_variant`** — the TEE variant that produced the verified quote, serialized as `TeeVariant`: `dstack-tdx`, `dstack-gcp-tdx`, `dstack-nitro-enclave`, `dstack-amd-sev-snp`, or `dstack-aws-nitro-tpm`. - **`acpi_tables_verified`** — whether TDX ACPI table contents were verified. This is useful for relying parties that require `requirements.tdx_measure_acpi_tables = true`. - **`key_provider`** — the decoded `app_info.key_provider_info` (`{name, id}`); `name` is e.g. `kms` or `local`. A `local` key provider means the CVM is not KMS-backed, which is itself a dev/insecure posture signal. The raw bytes remain in `app_info.key_provider_info`. -- **`boot_info`** — the policy object a relying party should feed to its auth/governance layer. For AWS EC2 NitroTPM this includes `attestationMode = dstack-aws-nitro-tpm`, PCR4/7/12-derived `osImageHash`, PCR14-bound `mrAggregated`, app identity, instance/device identity, and a `tcbStatus` normalized to `UpToDate`. +- **`boot_info`** — the policy object a relying party should feed to its auth/governance layer. For AWS EC2 NitroTPM this includes `teeVariant = dstack-aws-nitro-tpm`, PCR4/7/12-derived `osImageHash`, PCR14-bound `mrAggregated`, app identity, instance/device identity, and a `tcbStatus` normalized to `UpToDate`. `os_image_is_dev` and `os_image_version` are read from the image's `metadata.json`, which is part of `sha256sum.txt` and therefore bound to the `os_image_hash` that step 3 verifies against the quote — so they are as trustworthy as the os-image-hash check itself. They are `null` when the platform does not expose them (e.g. GCP TDX / Nitro Enclave) or when the image predates the field (images without `is_dev` are always production). diff --git a/dstack/verifier/builder/Dockerfile b/dstack/verifier/builder/Dockerfile index f8ad5274a..23940089b 100644 --- a/dstack/verifier/builder/Dockerfile +++ b/dstack/verifier/builder/Dockerfile @@ -9,7 +9,7 @@ ARG DSTACK_REV ARG DSTACK_SRC_URL=https://github.com/Dstack-TEE/dstack.git WORKDIR /build RUN ./pin-packages.sh ./builder-pinned-packages.txt -RUN apt-get update && \ +RUN apt-get update -o APT::Update::Error-Mode=any && \ apt-get install -y --no-install-recommends \ git \ build-essential \ @@ -30,57 +30,23 @@ RUN rustup target add x86_64-unknown-linux-musl RUN cd repo/dstack && cargo build --release -p dstack-verifier --target x86_64-unknown-linux-musl RUN echo "${DSTACK_REV}" > /build/.GIT_REV -FROM debian:bookworm@sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe AS acpi-builder -COPY --from=build-shared pin-packages.sh config-qemu.sh /build/ -COPY builder/shared/qemu-pinned-packages.txt /build/ -WORKDIR /build -ARG QEMU_REV=dbcec07c0854bf873d346a09e87e4c993ccf2633 -RUN ./pin-packages.sh ./qemu-pinned-packages.txt && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - git \ - libslirp-dev \ - python3-pip \ - ninja-build \ - pkg-config \ - libglib2.0-dev \ - python3-sphinx \ - python3-sphinx-rtd-theme \ - build-essential \ - flex \ - bison && \ - rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache -RUN git clone https://github.com/kvinwang/qemu-tdx.git --depth 1 --branch dstack-qemu-9.2.1 --single-branch && \ - cd qemu-tdx && git fetch --depth 1 origin ${QEMU_REV} && \ - git checkout ${QEMU_REV} && \ - ../config-qemu.sh ./build /usr/local && \ - cd build && \ - ninja && \ - strip qemu-system-x86_64 && \ - install -m 755 qemu-system-x86_64 /usr/local/bin/dstack-acpi-tables && \ - cd ../ && \ - install -d /usr/local/share/qemu && \ - install -m 644 pc-bios/efi-virtio.rom /usr/local/share/qemu/ && \ - install -m 644 pc-bios/kvmvapic.bin /usr/local/share/qemu/ && \ - install -m 644 pc-bios/linuxboot_dma.bin /usr/local/share/qemu/ && \ - cd .. && rm -rf qemu-tdx - FROM debian:bookworm@sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe COPY --from=build-shared pin-packages.sh /build/ COPY builder/shared/pinned-packages.txt /build/ WORKDIR /build RUN ./pin-packages.sh ./pinned-packages.txt && \ - apt-get update && \ + apt-get update -o APT::Update::Error-Mode=any && \ apt-get install -y --no-install-recommends \ ca-certificates \ curl \ - libglib2.0-0 \ - libslirp0 \ && rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache COPY --from=verifier-builder /build/repo/dstack/target/x86_64-unknown-linux-musl/release/dstack-verifier /usr/local/bin/dstack-verifier COPY --from=verifier-builder /build/.GIT_REV /etc/ -COPY --from=acpi-builder /usr/local/bin/dstack-acpi-tables /usr/local/bin/dstack-acpi-tables -COPY --from=acpi-builder /usr/local/share/qemu /usr/local/share/qemu +# The OCI metadata is generated once by build-lib.sh and injected here, as +# the config labels and as the manifest annotations, so the three views of +# the image can never disagree. +ARG IMAGE_METADATA +RUN mkdir -p /etc/dstack-verifier && printf '%s\n' "${IMAGE_METADATA}" > /etc/dstack-verifier/build-info RUN mkdir -p /etc/dstack COPY dstack-verifier.toml /etc/dstack/dstack-verifier.toml WORKDIR /var/lib/dstack-verifier diff --git a/dstack/verifier/builder/README.md b/dstack/verifier/builder/README.md new file mode 100644 index 000000000..9e409e925 --- /dev/null +++ b/dstack/verifier/builder/README.md @@ -0,0 +1,89 @@ +# dstack Verifier Builder + +This directory contains the files needed to build the dstack-verifier Docker +image reproducibly. + +## Overview + +The builder produces a Docker image containing the dstack-verifier service +compiled from Rust source. The build pins its base images by digest, pins the +Debian package set (`shared/pinned-packages.txt`), and normalizes timestamps, so +the same revision always yields the same image. + +## Prerequisites + +- Docker with Buildx v0.13.0+ (the script creates a BuildKit v0.20.2 builder) +- Git + +## Building the Image + +```bash +./build-image.sh <image-name>[:<tag>]... +``` + +For example: + +```bash +./build-image.sh my-org/dstack-verifier:dev +``` + +Optional environment variables: + +| Variable | Purpose | +| --- | --- | +| `GIT_REV` | Revision to build (default `HEAD`) | +| `IMAGE_VERSION` | Version recorded in the image metadata (default `dev`) | +| `IMAGE_SOURCE_URL` | Repository URL recorded in the image metadata | +| `NO_CACHE` | Set to any value to build without the layer cache | +| `OCI_TAR` | Also write an OCI archive here, for digest comparison | +| `METADATA_FILE` | Write the validated OCI manifest digest and build metadata here | +| `PUSH` | Set to any value to push the tags instead of only loading them | + +Publication and OCI export happen only after both package lists pass validation. +`NO_CACHE` applies to the validation builds; export then reuses their cached result. +Manual release workflows require an existing component release tag and check out +that tag, rather than building the branch selected in the workflow UI. + +## Reproducing a released image + +Release CI runs this same script, so a published image can be rebuilt and +checked digest-for-digest. Use a clean checkout of the release commit so the +Dockerfile, package lists, shared scripts and copied files also match the release. +From the repository root (fetch the tag first if it is not available locally): + +```bash +git switch --detach "verifier-v0.6.0^{commit}" +cd dstack/verifier/builder + +GIT_REV=HEAD \ +IMAGE_VERSION=0.6.0 \ +IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack \ +OCI_TAR=/tmp/verifier.oci.tar \ + ./build-image.sh dstacktee/dstack-verifier:0.6.0 + +python3 -c 'import json,tarfile;t=tarfile.open("/tmp/verifier.oci.tar");print(json.load(t.extractfile("index.json"))["manifests"][0]["digest"])' +``` + +`IMAGE_VERSION` is part of the image metadata, so it must match the release for +the digests to match. The printed digest is what the registry reports for +`dstacktee/dstack-verifier:0.6.0`; compare it with: + +```bash +docker buildx imagetools inspect dstacktee/dstack-verifier:0.6.0 --format '{{.Manifest.Digest}}' +``` + +Note that the release also publishes a `:latest` tag pointing at the same +digest. + +## Image metadata + +The image carries its provenance as OCI metadata in three places, all generated +from one definition in `dstack/build/shared/build-lib.sh`: + +- config labels — `docker inspect -f '{{json .Config.Labels}}' <image>` +- manifest annotations — `docker buildx imagetools inspect <image>` +- `/etc/dstack-verifier/build-info` inside the image, readable from within the CVM + +## Running the Built Image + +See the [verifier README](../README.md) for configuration and the HTTP API. diff --git a/dstack/verifier/builder/build-image.sh b/dstack/verifier/builder/build-image.sh index df3bfddbd..3530a80b0 100755 --- a/dstack/verifier/builder/build-image.sh +++ b/dstack/verifier/builder/build-image.sh @@ -11,29 +11,32 @@ REPO_ROOT=$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel) CONTEXT_DIR=$(dirname "$SCRIPT_DIR") SHARED_DIR="$SCRIPT_DIR/shared" DOCKERFILE="$SCRIPT_DIR/Dockerfile" +export CONTEXT_DIR DOCKERFILE +# shellcheck source=/dev/null source "$REPO_ROOT/dstack/build/shared/build-lib.sh" -NAME=${1:-} -if [ -z "$NAME" ]; then - echo "Usage: $0 <image-name>[:<tag>]" >&2 +if [ "$#" -eq 0 ]; then + echo "Usage: $0 <image-name>[:<tag>]..." >&2 exit 1 fi +TAGS=$(printf '%s\n' "$@") NO_CACHE=${NO_CACHE:-} GIT_REV=${GIT_REV:-HEAD} -GIT_REV=$(git -C "$REPO_ROOT" rev-parse "$GIT_REV") +GIT_REV=$(git -C "$REPO_ROOT" rev-parse --verify "${GIT_REV}^{commit}") DSTACK_SRC_URL=${DSTACK_SRC_URL:-https://github.com/Dstack-TEE/dstack.git} ensure_buildkit mkdir -p "$SHARED_DIR" touch "$SHARED_DIR/builder-pinned-packages.txt" -touch "$SHARED_DIR/qemu-pinned-packages.txt" touch "$SHARED_DIR/pinned-packages.txt" -docker_build "$NAME" "" "$SHARED_DIR/pinned-packages.txt" -docker_build "verifier-builder-temp" "verifier-builder" "$SHARED_DIR/builder-pinned-packages.txt" -docker_build "verifier-acpi-builder-temp" "acpi-builder" "$SHARED_DIR/qemu-pinned-packages.txt" +METADATA=$(image_metadata \ + "dstack-verifier" \ + "Remote attestation verification service for dstack" \ + "dstack/verifier" \ + "dstack/verifier/README.md") -check_clean_tree "$SHARED_DIR" +build_component "$TAGS" "verifier-builder-temp" "verifier-builder" "$SHARED_DIR" "$METADATA" diff --git a/dstack/verifier/builder/shared/pinned-packages.txt b/dstack/verifier/builder/shared/pinned-packages.txt index 286103f56..a99c0061b 100644 --- a/dstack/verifier/builder/shared/pinned-packages.txt +++ b/dstack/verifier/builder/shared/pinned-packages.txt @@ -42,7 +42,6 @@ libext2fs2:amd64=1.47.0-2 libffi8:amd64=3.4.4-1 libgcc-s1:amd64=12.2.0-14+deb12u1 libgcrypt20:amd64=1.10.1-3 -libglib2.0-0:amd64=2.74.6-2+deb12u8 libgmp10:amd64=2:6.2.1+dfsg1-1.1 libgnutls30:amd64=3.7.9-2+deb12u4 libgpg-error0:amd64=1.46-1 @@ -75,7 +74,6 @@ libselinux1:amd64=3.4-1+b6 libsemanage-common=3.4-1 libsemanage2:amd64=3.4-1+b5 libsepol2:amd64=3.4-2.1 -libslirp0:amd64=4.7.0-1 libsmartcols1:amd64=2.38.1-5+deb12u3 libss2:amd64=1.47.0-2 libssh2-1:amd64=1.10.0-3+b1 diff --git a/dstack/verifier/builder/shared/qemu-pinned-packages.txt b/dstack/verifier/builder/shared/qemu-pinned-packages.txt deleted file mode 100644 index 8cb00cf86..000000000 --- a/dstack/verifier/builder/shared/qemu-pinned-packages.txt +++ /dev/null @@ -1,236 +0,0 @@ -adduser=3.134 -apt=2.6.1 -base-files=12.4+deb12u11 -base-passwd=3.6.1 -bash=5.2.15-2+b8 -binutils-common:amd64=2.40-2 -binutils-x86-64-linux-gnu=2.40-2 -binutils=2.40-2 -bison=2:3.8.2+dfsg-1+b1 -bsdutils=1:2.38.1-5+deb12u3 -build-essential=12.9 -bzip2=1.0.8-5+b1 -ca-certificates=20230311+deb12u1 -coreutils=9.1-1 -cpp-12=12.2.0-14+deb12u1 -cpp=4:12.2.0-3 -dash=0.5.12-2 -debconf=1.5.82 -debian-archive-keyring=2023.3+deb12u2 -debianutils=5.7-0.5~deb12u1 -diffutils=1:3.8-4 -docutils-common=0.19+dfsg-6 -dpkg-dev=1.21.22 -dpkg=1.21.22 -e2fsprogs=1.47.0-2 -findutils=4.9.0-4 -flex=2.6.4-8.2 -fonts-font-awesome=5.0.10+really4.7.0~dfsg-4.1 -fonts-lato=2.0-2.1 -g++-12=12.2.0-14+deb12u1 -g++=4:12.2.0-3 -gcc-12-base:amd64=12.2.0-14+deb12u1 -gcc-12=12.2.0-14+deb12u1 -gcc=4:12.2.0-3 -git-man=1:2.39.5-0+deb12u3 -git=1:2.39.5-0+deb12u3 -gpgv=2.2.40-1.1 -grep=3.8-5 -gzip=1.12-1 -hostname=3.23+nmu1 -init-system-helpers=1.65.2 -libacl1:amd64=2.3.1-3 -libapt-pkg6.0:amd64=2.6.1 -libasan8:amd64=12.2.0-14+deb12u1 -libatomic1:amd64=12.2.0-14+deb12u1 -libattr1:amd64=1:2.5.1-4 -libaudit-common=1:3.0.9-1 -libaudit1:amd64=1:3.0.9-1 -libbinutils:amd64=2.40-2 -libblkid-dev:amd64=2.38.1-5+deb12u3 -libblkid1:amd64=2.38.1-5+deb12u3 -libbrotli1:amd64=1.0.9-2+b6 -libbz2-1.0:amd64=1.0.8-5+b1 -libc-bin=2.36-9+deb12u13 -libc-dev-bin=2.36-9+deb12u13 -libc6-dev:amd64=2.36-9+deb12u13 -libc6:amd64=2.36-9+deb12u13 -libcap-ng0:amd64=0.8.3-1+b3 -libcap2:amd64=1:2.66-4+deb12u1 -libcc1-0:amd64=12.2.0-14+deb12u1 -libcom-err2:amd64=1.47.0-2 -libcrypt-dev:amd64=1:4.4.33-2 -libcrypt1:amd64=1:4.4.33-2 -libctf-nobfd0:amd64=2.40-2 -libctf0:amd64=2.40-2 -libcurl3-gnutls:amd64=7.88.1-10+deb12u14 -libdb5.3:amd64=5.3.28+dfsg2-1 -libdebconfclient0:amd64=0.270 -libdpkg-perl=1.21.22 -libelf1:amd64=0.188-2.1 -liberror-perl=0.17029-2 -libexpat1:amd64=2.5.0-1+deb12u2 -libext2fs2:amd64=1.47.0-2 -libffi-dev:amd64=3.4.4-1 -libffi8:amd64=3.4.4-1 -libgcc-12-dev:amd64=12.2.0-14+deb12u1 -libgcc-s1:amd64=12.2.0-14+deb12u1 -libgcrypt20:amd64=1.10.1-3 -libgdbm-compat4:amd64=1.23-3 -libgdbm6:amd64=1.23-3 -libglib2.0-0:amd64=2.74.6-2+deb12u8 -libglib2.0-bin=2.74.6-2+deb12u8 -libglib2.0-data=2.74.6-2+deb12u8 -libglib2.0-dev-bin=2.74.6-2+deb12u8 -libglib2.0-dev:amd64=2.74.6-2+deb12u8 -libgmp10:amd64=2:6.2.1+dfsg1-1.1 -libgnutls30:amd64=3.7.9-2+deb12u4 -libgomp1:amd64=12.2.0-14+deb12u1 -libgpg-error0:amd64=1.46-1 -libgprofng0:amd64=2.40-2 -libgssapi-krb5-2:amd64=1.20.1-2+deb12u4 -libhogweed6:amd64=3.8.1-2 -libidn2-0:amd64=2.3.3-1+b1 -libisl23:amd64=0.25-1.1 -libitm1:amd64=12.2.0-14+deb12u1 -libjansson4:amd64=2.14-2 -libjs-jquery=3.6.1+dfsg+~3.5.14-1 -libjs-sphinxdoc=5.3.0-4 -libjs-underscore=1.13.4~dfsg+~1.11.4-3 -libjson-perl=4.10000-1 -libk5crypto3:amd64=1.20.1-2+deb12u4 -libkeyutils1:amd64=1.6.3-2 -libkrb5-3:amd64=1.20.1-2+deb12u4 -libkrb5support0:amd64=1.20.1-2+deb12u4 -libldap-2.5-0:amd64=2.5.13+dfsg-5 -liblsan0:amd64=12.2.0-14+deb12u1 -liblz4-1:amd64=1.9.4-1 -liblzma5:amd64=5.4.1-1 -libmd0:amd64=1.0.4-2 -libmount-dev:amd64=2.38.1-5+deb12u3 -libmount1:amd64=2.38.1-5+deb12u3 -libmpc3:amd64=1.3.1-1 -libmpfr6:amd64=4.2.0-1 -libncursesw6:amd64=6.4-4 -libnettle8:amd64=3.8.1-2 -libnghttp2-14:amd64=1.52.0-1+deb12u2 -libnsl-dev:amd64=1.3.0-2 -libnsl2:amd64=1.3.0-2 -libp11-kit0:amd64=0.24.1-2 -libpam-modules-bin=1.5.2-6+deb12u1 -libpam-modules:amd64=1.5.2-6+deb12u1 -libpam-runtime=1.5.2-6+deb12u1 -libpam0g:amd64=1.5.2-6+deb12u1 -libpcre2-16-0:amd64=10.42-1 -libpcre2-32-0:amd64=10.42-1 -libpcre2-8-0:amd64=10.42-1 -libpcre2-dev:amd64=10.42-1 -libpcre2-posix3:amd64=10.42-1 -libperl5.36:amd64=5.36.0-7+deb12u3 -libpkgconf3:amd64=1.8.1-1 -libpsl5:amd64=0.21.2-1 -libpython3-stdlib:amd64=3.11.2-1+b1 -libpython3.11-minimal:amd64=3.11.2-6+deb12u6 -libpython3.11-stdlib:amd64=3.11.2-6+deb12u6 -libquadmath0:amd64=12.2.0-14+deb12u1 -libreadline8:amd64=8.2-1.3 -librtmp1:amd64=2.4+20151223.gitfa8646d.1-2+b2 -libsasl2-2:amd64=2.1.28+dfsg-10 -libsasl2-modules-db:amd64=2.1.28+dfsg-10 -libseccomp2:amd64=2.5.4-1+deb12u1 -libselinux1-dev:amd64=3.4-1+b6 -libselinux1:amd64=3.4-1+b6 -libsemanage-common=3.4-1 -libsemanage2:amd64=3.4-1+b5 -libsepol-dev:amd64=3.4-2.1 -libsepol2:amd64=3.4-2.1 -libslirp-dev:amd64=4.7.0-1 -libslirp0:amd64=4.7.0-1 -libsmartcols1:amd64=2.38.1-5+deb12u3 -libsqlite3-0:amd64=3.40.1-2+deb12u2 -libss2:amd64=1.47.0-2 -libssh2-1:amd64=1.10.0-3+b1 -libssl3:amd64=3.0.18-1~deb12u2 -libstdc++-12-dev:amd64=12.2.0-14+deb12u1 -libstdc++6:amd64=12.2.0-14+deb12u1 -libsystemd0:amd64=252.38-1~deb12u1 -libtasn1-6:amd64=4.19.0-2+deb12u1 -libtinfo6:amd64=6.4-4 -libtirpc-common=1.3.3+ds-1 -libtirpc-dev:amd64=1.3.3+ds-1 -libtirpc3:amd64=1.3.3+ds-1 -libtsan2:amd64=12.2.0-14+deb12u1 -libubsan1:amd64=12.2.0-14+deb12u1 -libudev1:amd64=252.38-1~deb12u1 -libunistring2:amd64=1.0-2 -libuuid1:amd64=2.38.1-5+deb12u3 -libxxhash0:amd64=0.8.1-1 -libzstd1:amd64=1.5.4+dfsg2-5 -linux-libc-dev:amd64=6.1.164-1 -login=1:4.13+dfsg1-1+deb12u1 -logsave=1.47.0-2 -m4=1.4.19-3 -make=4.3-4.1 -mawk=1.3.4.20200120-3.1 -media-types=10.0.0 -mount=2.38.1-5+deb12u3 -ncurses-base=6.4-4 -ncurses-bin=6.4-4 -ninja-build=1.11.1-2~deb12u1 -openssl=3.0.18-1~deb12u2 -passwd=1:4.13+dfsg1-1+deb12u1 -patch=2.7.6-7 -perl-base=5.36.0-7+deb12u3 -perl-modules-5.36=5.36.0-7+deb12u3 -perl=5.36.0-7+deb12u3 -pkg-config:amd64=1.8.1-1 -pkgconf-bin=1.8.1-1 -pkgconf:amd64=1.8.1-1 -python-babel-localedata=2.10.3-1 -python3-alabaster=0.7.12-1 -python3-babel=2.10.3-1 -python3-certifi=2022.9.24-1 -python3-chardet=5.1.0+dfsg-2 -python3-charset-normalizer=3.0.1-2 -python3-distutils=3.11.2-3 -python3-docutils=0.19+dfsg-6 -python3-idna=3.3-1+deb12u1 -python3-imagesize=1.4.1-1 -python3-jinja2=3.1.2-1+deb12u3 -python3-lib2to3=3.11.2-3 -python3-markupsafe=2.1.2-1+b1 -python3-minimal=3.11.2-1+b1 -python3-packaging=23.0-1 -python3-pip=23.0.1+dfsg-1 -python3-pkg-resources=66.1.1-1+deb12u2 -python3-pygments=2.14.0+dfsg-1 -python3-requests=2.28.1+dfsg-1 -python3-roman=3.3-3 -python3-setuptools=66.1.1-1+deb12u2 -python3-six=1.16.0-4 -python3-snowballstemmer=2.2.0-2 -python3-sphinx-rtd-theme=1.2.0+dfsg-1 -python3-sphinx=5.3.0-4 -python3-tz=2022.7.1-4 -python3-urllib3=1.26.12-1+deb12u3 -python3-wheel=0.38.4-2 -python3.11-minimal=3.11.2-6+deb12u6 -python3.11=3.11.2-6+deb12u6 -python3=3.11.2-1+b1 -readline-common=8.2-1.3 -rpcsvc-proto=1.4.3-1 -sed=4.9-1 -sgml-base=1.31 -sphinx-common=5.3.0-4 -sphinx-rtd-theme-common=1.2.0+dfsg-1 -sysvinit-utils=3.06-4 -tar=1.34+dfsg-1.2+deb12u1 -tzdata=2025b-0+deb12u1 -usr-is-merged=37~deb12u1 -util-linux-extra=2.38.1-5+deb12u3 -util-linux=2.38.1-5+deb12u3 -uuid-dev:amd64=2.38.1-5+deb12u3 -xml-core=0.18+nmu1 -xz-utils=5.4.1-1 -zlib1g-dev:amd64=1:1.2.13.dfsg-1 -zlib1g:amd64=1:1.2.13.dfsg-1 diff --git a/dstack/verifier/dstack-verifier.toml b/dstack/verifier/dstack-verifier.toml index b098c533a..bdb09fe92 100644 --- a/dstack/verifier/dstack-verifier.toml +++ b/dstack/verifier/dstack-verifier.toml @@ -15,5 +15,18 @@ image_download_url = "https://download.dstack.org/os-images/mr_{OS_IMAGE_HASH}.t # Image download timeout in seconds image_download_timeout_secs = 300 -# Optional PCCS URL for quote verification -# pccs_url = "https://pccs.phala.network" +[attestation] +insecure_allow_external_trust_anchors = false + +[attestation.urls] +# pccs = "https://pccs.phala.network" +# amd_kds = "https://kdsintf.amd.com/vcek/v1" + +[attestation.root_ca] +# tdx = "/etc/dstack/roots/intel-sgx-root-ca.der" +# gcp_tpm = "/etc/dstack/roots/gcp-tpm-root-ca.pem" +# aws_nitro_enclave = "/etc/dstack/roots/aws-nitro-enclave-root-ca.pem" +# aws_nitro_tpm = "/etc/dstack/roots/aws-nitro-tpm-root-ca.pem" +# sev_snp_milan = "/etc/dstack/roots/amd-milan-ark.pem" +# sev_snp_genoa = "/etc/dstack/roots/amd-genoa-ark.pem" +# sev_snp_turin = "/etc/dstack/roots/amd-turin-ark.pem" diff --git a/dstack/verifier/fixtures/tdx-lite-attestation.json b/dstack/verifier/fixtures/tdx-lite-attestation.json index 93f4af0ab..2b5651621 100644 --- a/dstack/verifier/fixtures/tdx-lite-attestation.json +++ b/dstack/verifier/fixtures/tdx-lite-attestation.json @@ -1,3 +1,3 @@ { - "attestation": "0000394e040002008100000000000000939a7233f79c4ca9940a0db3957f06071026ff2bbebac59cc1ef911279d9481b000000000c010400000000000000000000000000d0d80c085166ba78ccc69af268e5753cf0f3394523cb4ff7c50b08d9265c82489c099c377be6a400e4d2b57da924012c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000e702060000000000fd685522ce791dfef67414614eb07d03fc07a32c5a66f36288b329dab92b724b1564c73d436ffb9ea84488c51ac5a1c50186b0e55f2fa8e4fb69d890f14f54d5612707646e2573d54e0d2ddaaade77caa9000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f8438db36b96f85d8752ff7f24a89ec05c79ec9eda2ba732c897fb970ca429365b7471b1c054cb84f17b1c2b23ba66402023546e7f3b9d1228e274f70c44d481162540f8452544520a796a52f06879709b81a824a26792a7822327504b0d2aee4c1b739ed451a637b0f82642e48a5ea83925d23633c72e7385c8e9aca4175e133ed1625b7d92eb39edf509c27ff392dc6f24c170d0fd63fc2b1b53202eea47b013978437fa6982cf5e0438ff95c208994aaa0f4ebab2e3a66824b5b56869137e646970313a3a736563703235366b31632d706b3a41353570576d74654a494a4f6a385f7049372d707a654478793147327131384744763838484e526442586b51cc1000008bca152d0454bdfd5adab1bc3a527884f77ea7993d32ee0e4426b2ae0fe42bf3f5642d6abd763b4f4c6042133e2ed79cce743f2c54ff4c7ea5d712dc1172ec244fe5b32ac6ffeb104614bcb8894c7aaafbbbe6f6bfd852f5dcd6cf400557ee764e62850d955975d93eff63b17e6e13e329a7bb13926706c0430017d543ab01920600461000000404191b04ff0006000000000000000000000000000000000000000000000000000000000000000000000000000000001500000000000000e700000000000000e5a3a7b5d830c2953b98534c6c59a3a34fdc34e933f7f5898f0a85cf08846bca0000000000000000000000000000000000000000000000000000000000000000dc9e2a7c6f948f17474e34a7fc43ed030f7c1563f1babddf6340c82e0e54a8c500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000600000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c1140324365c08f021a721dbe9175cb89dcd2235e2bd00bfb235b2a66b8c783600000000000000000000000000000000000000000000000000000000000000002af8cd12d44e0d22f904b15c02968b57b668e7f2487ba308e1d9a269ea125e48b243f7d32bb8551e1e3c2c09bd2162d36941eeb47be50b9b55a766a14d0cfe302000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f05005e0e00002d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d49494538444343424a6167417749424167495556706163774c766c316d476155506b384b4375504141334769465177436759494b6f5a497a6a3045417749770a634445694d434147413155454177775a535735305a577767553064594946424453794251624746305a6d397962534244515445614d42674741315545436777520a535735305a577767513239796347397959585270623234784644415342674e564241634d43314e68626e526849454e7359584a684d51737743515944565151490a44414a445154454c4d416b474131554542684d4356564d774868634e4d6a59774e4445314d4441314d4455345768634e4d7a4d774e4445314d4441314d4455340a576a42774d534977494159445651514444426c4a626e526c624342545231676755454e4c49454e6c636e52705a6d6c6a5958526c4d526f77474159445651514b0a4442464a626e526c6243424462334a7762334a6864476c76626a45554d424947413155454277774c553246756447456751327868636d4578437a414a42674e560a4241674d416b4e424d517377435159445651514745774a56557a425a4d424d4742797147534d34394167454743437147534d343941774548413049414245586a0a53374265726c3262726b65543677707878436a556536564775577268586e51767a41395862524768356b68637671766b566b427874715935475759544f6551340a5948496a636b7974734c6c5531774b594a74576a67674d4d4d4949444344416642674e5648534d4547444157674253566231334e765276683655424a796454300a4d383442567776655644427242674e56485238455a4442694d47436758714263686c706f64485277637a6f764c32467761533530636e567a6447566b633256790a646d6c6a5a584d75615735305a577775593239744c334e6e6543396a5a584a3061575a7059324630615739754c3359304c33426a61324e796244396a595431770a624746305a6d397962535a6c626d4e765a476c755a7a316b5a584977485159445652304f42425945464362386b6b73714d364c384f6765734c713943337339440a7a5333504d41344741315564447745422f775145417749477744414d42674e5648524d4241663845416a41414d4949434f51594a4b6f5a496876684e415130420a424949434b6a4343416959774867594b4b6f5a496876684e4151304241515151514e367178312b487a7758704c373859496b716c646a434341574d47436971470a534962345451454e41514977676746544d42414743797147534962345451454e41514942416745454d42414743797147534962345451454e41514943416745450a4d42414743797147534962345451454e41514944416745434d42414743797147534962345451454e41514945416745434d42414743797147534962345451454e0a41514946416745454d42414743797147534962345451454e41514947416745424d42414743797147534962345451454e41514948416745414d424147437971470a534962345451454e41514949416745464d42414743797147534962345451454e4151494a416745414d42414743797147534962345451454e4151494b416745410a4d42414743797147534962345451454e4151494c416745414d42414743797147534962345451454e4151494d416745414d42414743797147534962345451454e0a4151494e416745414d42414743797147534962345451454e4151494f416745414d42414743797147534962345451454e41514950416745414d424147437971470a534962345451454e41514951416745414d42414743797147534962345451454e415149524167454e4d42384743797147534962345451454e41514953424241450a42414943424145414251414141414141414141414d42414743697147534962345451454e41514d45416741414d42514743697147534962345451454e415151450a42704441627741414144415042676f71686b69472b45304244514546436745424d42344743697147534962345451454e4151594545464a37386f7137314543670a6c7536335265417a675430775241594b4b6f5a496876684e41513042427a41324d42414743797147534962345451454e415163424151482f4d424147437971470a534962345451454e41516343415145414d42414743797147534962345451454e415163444151482f4d416f4743437147534d343942414d43413067414d4555430a494778676472434e7a344753716d32647a4c45533874757663717230444d692b427537533771537133325343416945417439454f6377584f6a31484a4c4462750a6d473357414549577962624f61635959612b7253384366526c514d3d0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436c6a4343416a32674177494241674956414a567658633239472b487051456e4a3150517a7a674658433935554d416f4743437147534d343942414d430a4d476778476a415942674e5642414d4d45556c756447567349464e48574342536232393049454e424d526f77474159445651514b4442464a626e526c624342440a62334a7762334a6864476c76626a45554d424947413155454277774c553246756447456751327868636d4578437a414a42674e564241674d416b4e424d5173770a435159445651514745774a56557a4165467730784f4441314d6a45784d4455774d5442614677307a4d7a41314d6a45784d4455774d5442614d484178496a41670a42674e5642414d4d47556c756447567349464e4857434251513073675547786864475a76636d306751304578476a415942674e5642416f4d45556c75644756730a49454e76636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b474131554543417743513045780a437a414a42674e5642415954416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a304441516344516741454e53422f377432316c58534f0a3243757a7078773734654a423732457944476757357258437478327456544c7136684b6b367a2b5569525a436e71523770734f766771466553786c6d546c4a6c0a65546d693257597a33714f42757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f536347724442530a42674e5648523845537a424a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b633256790a646d6c6a5a584d75615735305a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e5648513445466751556c5739640a7a62306234656c4153636e553944504f4156634c336c517744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159420a4166384341514177436759494b6f5a497a6a30454177494452774177524149675873566b6930772b6936565947573355462f32327561586530594a446a3155650a6e412b546a44316169356343494359623153416d4435786b66545670766f34556f79695359787244574c6d5552344349394e4b7966504e2b0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436a7a4343416a53674177494241674955496d554d316c71644e496e7a6737535655723951477a6b6e42717777436759494b6f5a497a6a3045417749770a614445614d4267474131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e760a636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a0a42674e5642415954416c56544d423458445445344d4455794d5445774e4455784d466f58445451354d54497a4d54497a4e546b314f566f77614445614d4267470a4131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e76636e4276636d46300a615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a42674e56424159540a416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a3044415163445167414543366e45774d4449595a4f6a2f69505773437a61454b69370a314f694f534c52466857476a626e42564a66566e6b59347533496a6b4459594c304d784f346d717379596a6c42616c54565978465032734a424b357a6c4b4f420a757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f5363477244425342674e5648523845537a424a0a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b63325679646d6c6a5a584d75615735300a5a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e564851344546675155496d554d316c71644e496e7a673753560a55723951477a6b6e4271777744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159424166384341514577436759490a4b6f5a497a6a3045417749445351417752674968414f572f35516b522b533943695344634e6f6f774c7550524c735747662f59693747535839344267775477670a41694541344a306c72486f4d732b586f356f2f7358364f39515778485241765a55474f6452513763767152586171493d0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000030000000000a000000c0095d04cf26fe03aef6e3561fa24c1aa1cea93f4aeaf563b1f9f7616184c53454875925759434769cec2490acb563a3372c616370692d6c6f6164657224414350492044415441000000000a000000c08d9a4d4777a1bc77ecd9d8d37a4628129a80052a510320159a20a923bd07a0e90d8d1f2e1ebf088992b25f0d0fa672ef24616370692d7273647024414350492044415441000000000a000000c03070721e169bc41884724cb0e6b3082e1baf249083d8b389181ba50b9afa951057876c380b8870e8c2facf2eff67a2b62c616370692d7461626c6573244143504920444154410300000001000008004073797374656d2d707265706172696e6700030000000100000800186170702d69645086b0e55f2fa8e4fb69d890f14f54d5612707646e03000000010000080030636f6d706f73652d686173688086b0e55f2fa8e4fb69d890f14f54d5612707646e2573d54e0d2ddaaade77caa90300000001000008002c696e7374616e63652d696450050bf89570575fe8fab4cb8f0a62a9e64efe8ead03000000010000080030626f6f742d6d722d646f6e6500030000000100000800346f732d696d6167652d686173688007a2388c7a6a1b6a646d443f1517990a4ec294471d63146cda9d56972765051d030000000100000800306b65792d70726f766964657231037b226e616d65223a226b6d73222c226964223a223330353933303133303630373261383634386365336430323031303630383261383634386365336430333031303730333432303030343266373165323334643733333961316365616361303963336333393165623831366335333366393830616461616233346631366561643039336666306163313030643963303332353361333035366636643237373335313235343333313830623365363163353461373866336664313333333738363965303035316465653036227d0300000001000008002873746f726167652d66730c7a66730300000001000008003073797374656d2d726561647900244073797374656d2d707265706172696e6700186170702d69645086b0e55f2fa8e4fb69d890f14f54d5612707646e30636f6d706f73652d686173688086b0e55f2fa8e4fb69d890f14f54d5612707646e2573d54e0d2ddaaade77caa92c696e7374616e63652d696450050bf89570575fe8fab4cb8f0a62a9e64efe8ead30626f6f742d6d722d646f6e6500346f732d696d6167652d686173688007a2388c7a6a1b6a646d443f1517990a4ec294471d63146cda9d56972765051d306b65792d70726f766964657231037b226e616d65223a226b6d73222c226964223a223330353933303133303630373261383634386365336430323031303630383261383634386365336430333031303730333432303030343266373165323334643733333961316365616361303963336333393165623831366335333366393830616461616233346631366561643039336666306163313030643963303332353361333035366636643237373335313235343333313830623365363163353461373866336664313333333738363965303035316465653036227d2873746f726167652d66730c7a66733073797374656d2d726561647900646970313a3a736563703235366b31632d706b3a41353570576d74654a494a4f6a385f7049372d707a654478793147327131384744763838484e526442586b514d107b226f735f696d6167655f68617368223a2265366635636665633230633032653762393762616132313364306637313830323062353565303430313732643930636362636239343664353663386230396462222c226370755f636f756e74223a322c226d656d6f72795f73697a65223a323134373438333634382c2271656d755f76657273696f6e223a22382e322e32222c227063695f686f6c6536345f73697a65223a302c22687567657061676573223a66616c73652c226e756d5f67707573223a302c226e756d5f6e767377697463686573223a302c22686f74706c75675f6f6666223a66616c73652c22696d616765223a2264737461636b2d302e362e30222c22686f73745f73686172655f6d6f6465223a223970222c226f766d665f76617269616e74223a22707265323032353035222c227464785f6174746573746174696f6e5f76617269616e74223a226c697465222c227464785f6d6561737572656d656e74223a7b226d6561737572656d656e74223a226f3264325a584a7a61573975413256706257466e5a614e755932316b62476c755a56397a6147457a4f4452594d486869674951726332516f656a70773257392b4d4a4a5368587672526673666b54464b4c71686a32777263424d6844487376796d705a6b425742474d61577175484e725a584a755a577866595856306147567564476c6a6232526c57444373666d4d747a317a536f66356348304830326267686c58446d5474504745446a3976795641546d39554c2f3158386e6138554859776676723467756257515864746157357064484a6b58334e6f59544d344e466777542b54336351453070683139377a5636335772464339762b376c4179704d45414e3134676368622f35436f37315949724a4f5a352b5255422f2f6556754255685a48526b646d616a5a47393262575a7063484a6c4d6a41794e5441315a4731796447536961334e70626d64735a56397759584e7a5744436d3871795555594547687154624a5a2f6f2b6c51343345705976616e394c317366734a4b444e584256414e4b6146636b6a683046714c314c64334f6d63672f686f644864765833426863334e594d50316f56534c4f6552332b396e5155595536776651503842364d73576d627a596f697a4b6471354b334a4c4657544850554e762b35366f52496a464773576878575a305a46396f62324a4d6742414a424141474351494c41684151222c22636865636b73756d5f66696c65223a224f474d355a546b314e575977596a55334e324e6a4f446b304e5745344f5446684d7a526a59325a684e5749345a544134596d49315a54566b4d6a51354f546b794e6a417a5a5459305a4751784e4467334d47557a5979416762575668633356795a57316c626e5175644752344c6d4e696233494b227d2c22737065635f76657273696f6e223a317d" + "attestation": "0000394e040002008100000000000000939a7233f79c4ca9940a0db3957f0607dca47981c0d1095de780f491950bc3c9000000000e010400000000000000000000000000346bc77a1846cac214dd2e8edeb9ee4349449d6c3f9ff2c52149a634c27b7fd1bd314c2ef6b973eeebd55742952531a100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000e702060000000000f2032d37d482f3aaa3b482e7df8cd917eb8641d99d51bab64cff6e549d15f58eb4dc7f207365fd424a84935c79b8cb740160618cd0c23d15d921b788b8fc3a0d024ac72295185da3ba41d5a065bc7aa833000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f8438db36b96f85d8752ff7f24a89ec05c79ec9eda2ba732c897fb970ca429365b7471b1c054cb84f17b1c2b23ba66404ffc8bcd33abc1bbf1d772196dc6d9f68cbe64c07b16a74ac085aa22ae976731bbb34a59255f119ec1b2ba53018cfa8eea91d5b853964f711fd60c71d1d4a96ebe975801610a869ebedb95fcaa372d4961377d139a6f3a3b21c4ac4352f70748b49119652a84beeee90c614992c1f67461b17edac4a4f6034c72b147c51fae66920eb2d23e8d40287fcebf5ac2a263e842424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242cc1000006795ae441972a6b9735c67e0232aec16c29a66ee3bee7bb0a9a951e81aa0a96d7d168acda4d90b7991ef995d018d09066e5bad4009c3037d58fead5c5b5bdb40445e655dfb4499fb6e3eb434c3275c340df9efb896c56d628111c4b955c15bf31774c93727636fdbbc8236633bb83f08388c368684d3c75e29aed47ec050e7720600461000000404191b04ff0006000000000000000000000000000000000000000000000000000000000000000000000000000000001500000000000000e700000000000000e5a3a7b5d830c2953b98534c6c59a3a34fdc34e933f7f5898f0a85cf08846bca0000000000000000000000000000000000000000000000000000000000000000dc9e2a7c6f948f17474e34a7fc43ed030f7c1563f1babddf6340c82e0e54a8c500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000600000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c209022cccc4d771dc1ede10db23ac55be3419d22901a89fb8cc6138404337b2000000000000000000000000000000000000000000000000000000000000000018264bac729410b741866c05234532ec0ba2725d1a4b6328a73151cdd0681b4c395ea0fb11bca5e540cfa36967d094400ff61b54d12df313517bd02c88e15ab42000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f05005e0e00002d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d49494538444343424a65674177494241674956414b576652394d75626c4e5642566e4250454a58614b6466567151394d416f4743437147534d343942414d430a4d484178496a416742674e5642414d4d47556c756447567349464e4857434251513073675547786864475a76636d306751304578476a415942674e5642416f4d0a45556c756447567349454e76636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155450a4341774351304578437a414a42674e5642415954416c56544d423458445449324d4463794e6a41334e4445774f466f5844544d7a4d4463794e6a41334e4445770a4f466f77634445694d434147413155454177775a535735305a5777675530645949464244537942445a584a3061575a70593246305a5445614d426747413155450a43677752535735305a577767513239796347397959585270623234784644415342674e564241634d43314e68626e526849454e7359584a684d517377435159440a5651514944414a445154454c4d416b474131554542684d4356564d775754415442676371686b6a4f5051494242676771686b6a4f50514d4242774e43414151510a644b6d76382f774831684e6476705639474d476345442b514b764f382b6276676f752b426362774c432b79582f78427778384d33395444347262436636764a570a54786f442f496d61784b2b337857554864704d496f3449444444434341776777487759445652306a42426777466f41556c5739647a62306234656c4153636e550a3944504f4156634c336c5177617759445652306642475177596a42676f46366758495a616148523063484d364c79396863476b7564484a316333526c5a484e6c0a636e5a705932567a4c6d6c75644756734c6d4e766253397a5a3367765932567964476c6d61574e6864476c76626939324e4339775932746a636d772f593245390a6347786864475a76636d306d5a57356a62325270626d63395a4756794d42304741315564446751574242515a43536c546e6b372f446b746d702f504a6d304f500a7a417455357a414f42674e56485138424166384542414d434273417744415944565230544151482f4241497741444343416a6b4743537147534962345451454e0a4151534341696f776767496d4d42344743697147534962345451454e415145454547396738357947645143775644587367467967472b6f776767466a42676f710a686b69472b453042445145434d494942557a415142677371686b69472b45304244514543415149424244415142677371686b69472b45304244514543416749420a4244415142677371686b69472b4530424451454341774942416a415142677371686b69472b4530424451454342414942416a415142677371686b69472b4530420a44514543425149424244415142677371686b69472b45304244514543426749424154415142677371686b69472b453042445145434277494241444151426773710a686b69472b45304244514543434149424254415142677371686b69472b45304244514543435149424144415142677371686b69472b45304244514543436749420a4144415142677371686b69472b45304244514543437749424144415142677371686b69472b45304244514543444149424144415142677371686b69472b4530420a44514543445149424144415142677371686b69472b45304244514543446749424144415142677371686b69472b453042445145434477494241444151426773710a686b69472b45304244514543454149424144415142677371686b69472b45304244514543455149424454416642677371686b69472b45304244514543456751510a42415143416751424141554141414141414141414144415142676f71686b69472b45304244514544424149414144415542676f71686b69472b453042445145450a4241615177473841414141774477594b4b6f5a496876684e4151304242516f424154416542676f71686b69472b4530424451454742424238724731754c5a50690a3839584569693555414154684d45514743697147534962345451454e415163774e6a415142677371686b69472b45304244514548415145422f7a4151426773710a686b69472b45304244514548416745422f7a415142677371686b69472b45304244514548417745422f7a414b42676771686b6a4f5051514441674e48414442450a4169414a6d6b77376745482b636d2f4446667275664b3834374f7475546465322f7535615031732f42617237504149675364384c50315161743847506a6a34650a684c2b315470714e64557a2f5946733053515279545941466774343d0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436c6a4343416a32674177494241674956414a567658633239472b487051456e4a3150517a7a674658433935554d416f4743437147534d343942414d430a4d476778476a415942674e5642414d4d45556c756447567349464e48574342536232393049454e424d526f77474159445651514b4442464a626e526c624342440a62334a7762334a6864476c76626a45554d424947413155454277774c553246756447456751327868636d4578437a414a42674e564241674d416b4e424d5173770a435159445651514745774a56557a4165467730784f4441314d6a45784d4455774d5442614677307a4d7a41314d6a45784d4455774d5442614d484178496a41670a42674e5642414d4d47556c756447567349464e4857434251513073675547786864475a76636d306751304578476a415942674e5642416f4d45556c75644756730a49454e76636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b474131554543417743513045780a437a414a42674e5642415954416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a304441516344516741454e53422f377432316c58534f0a3243757a7078773734654a423732457944476757357258437478327456544c7136684b6b367a2b5569525a436e71523770734f766771466553786c6d546c4a6c0a65546d693257597a33714f42757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f536347724442530a42674e5648523845537a424a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b633256790a646d6c6a5a584d75615735305a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e5648513445466751556c5739640a7a62306234656c4153636e553944504f4156634c336c517744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159420a4166384341514177436759494b6f5a497a6a30454177494452774177524149675873566b6930772b6936565947573355462f32327561586530594a446a3155650a6e412b546a44316169356343494359623153416d4435786b66545670766f34556f79695359787244574c6d5552344349394e4b7966504e2b0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436a7a4343416a53674177494241674955496d554d316c71644e496e7a6737535655723951477a6b6e42717777436759494b6f5a497a6a3045417749770a614445614d4267474131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e760a636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a0a42674e5642415954416c56544d423458445445344d4455794d5445774e4455784d466f58445451354d54497a4d54497a4e546b314f566f77614445614d4267470a4131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e76636e4276636d46300a615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a42674e56424159540a416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a3044415163445167414543366e45774d4449595a4f6a2f69505773437a61454b69370a314f694f534c52466857476a626e42564a66566e6b59347533496a6b4459594c304d784f346d717379596a6c42616c54565978465032734a424b357a6c4b4f420a757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f5363477244425342674e5648523845537a424a0a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b63325679646d6c6a5a584d75615735300a5a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e564851344546675155496d554d316c71644e496e7a673753560a55723951477a6b6e4271777744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159424166384341514577436759490a4b6f5a497a6a3045417749445351417752674968414f572f35516b522b533943695344634e6f6f774c7550524c735747662f59693747535839344267775477670a41694541344a306c72486f4d732b586f356f2f7358364f39515778485241765a55474f6452513763767152586171493d0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000074000000000b000080c00b8772e5b0b41b83e6044a68397e02f49fb47066b4fbe4917ea2c45c64f323fdacbb37948f821ebaf8bc9c938ba8a74900a8095464785461626c65000100000000000000af96bb93f2b9b84e9462e0ba745642360090800000000000000000000a000080c0344bc51c980ba621aaa00da3ed7436f7d6e549197dfe699515dfa2c6583d95e6412af21c097d473155875ffd561d679000e82946762858585858585858582d585858582d585858582d585858582d58585858585858585858585829000000c0ff0000000000400800000000000000000001000080c09dc3a1f80bcec915391dcda5ffbb15e7419f77eab462bbf72b42166fb70d50325e37b36f93537a863769bcf9bedae6fb00d061dfe48bca93d211aa0d00e098032b8c0a00000000000000000000000000000053006500630075007200650042006f006f0074000000000001000080c06f2e3cbc14f9def86980f5f66fd85e99d63e69a73014ed8a5633ce56eca5b64b692108c56110e22acadcef58c3250f1b009061dfe48bca93d211aa0d00e098032b8c0200000000000000000000000000000050004b000000000001000080c0d607c0efb41c0d757d69bca0615c3a9ac0b1db06c557d992e906c6b7dee40e0e031640c7bfd7bcd35844ef9edeadc6f9009861dfe48bca93d211aa0d00e098032b8c030000000000000000000000000000004b0045004b000000000001000080c008a74f8963b337acb6c93682f934496373679dd26af1089cb4eaf0c30cf260a12e814856385ab8843e56a9acea19e1270090cbb219d73a3d9645a3bcdad00e67656f02000000000000000000000000000000640062000000000001000080c018cc6e01f0c6ea99aa23f8a280423e94ad81d96d0aeb5180504fc0f7a40cb3619dd39bd6a95ec1680a86ed6ab0f9828d0098cbb219d73a3d9645a3bcdad00e67656f030000000000000000000000000000006400620078000000000004000000c0394341b7182cd227c5c6b07ef8000cdfd86136c4292b8e576573ad7ed9ae41019f5818b4b971c9effc60e1ad9f1289f0001000000000000000000a000000c0095d04cf26fe03aef6e3561fa24c1aa1cea93f4aeaf563b1f9f7616184c53454875925759434769cec2490acb563a3372c616370692d6c6f6164657224414350492044415441000000000a000000c08d9a4d4777a1bc77ecd9d8d37a4628129a80052a510320159a20a923bd07a0e90d8d1f2e1ebf088992b25f0d0fa672ef24616370692d7273647024414350492044415441000000000a000000c03070721e169bc41884724cb0e6b3082e1baf249083d8b389181ba50b9afa951057876c380b8870e8c2facf2eff67a2b62c616370692d7461626c6573244143504920444154410100000003000080c0a6dc51e745a0e537afddfc7e06567109692e7840f0334f0392affea35039d10bbb460d86c7310c82de8029a111816ef000290118007c7a0000000000b2d0000000000000000000000000002a000000000000000403140072f728144ab61e44b8c39ebdd7f893c7040412006b00650072006e0065006c0000007fff04000000000002000080c01dd6f7b457ad880d840d41c961283bab688e94e4b59359ea45686581e90feccea3c624b1226113f824f315eb60ae0a7c00d061dfe48bca93d211aa0d00e098032b8c0900000000000000020000000000000042006f006f0074004f00720064006500720000000000000002000080c023ada07f5261f12f34a0bd8e46760962d6b4d576a416f1fea1c64bc656b1d28eacf7047ae6e967c58fd2a98bfa74c29800b90161dfe48bca93d211aa0d00e098032b8c08000000000000003e0000000000000042006f006f0074003000300030003000090100002c0055006900410070007000000004071400c9bdb87cebf8344faaea3ee4af6516a10406140021aa2c4614760345836e8ab6f46623317fff04000100000007000080c077a0dab2312b4e1e57a84d865a21e5b2ee8d677a21012ada819d0a98988078d3d740f6346bfe0abaa938ca20439a8d7100a043616c6c696e6720454649204170706c69636174696f6e2066726f6d20426f6f74204f7074696f6e0100000004000000c0394341b7182cd227c5c6b07ef8000cdfd86136c4292b8e576573ad7ed9ae41019f5818b4b971c9effc60e1ad9f1289f00010000000000200000006000000c0fbdcf9ef5e402595d8e2c9c412e3b2839d480d4a145e8347d1ebc8a165d920f55c62221c7f2ac3bfba1b80e5a1f3fc6e0088ed223b8f1a0000004c4f414445445f494d4147453a3a4c6f61644f7074696f6e73000200000006000000c04e34fcea0e56bc93582bb3acd0b7c9ed7b3cb41a8c59826dada5033e2ff3c7cdcb7b346542d033642581454da2726ec30054ec223b8f0d0000004c696e757820696e69747264000100000007000080c0214b0bef1379756011344877743fdc2a5382bac6e70362d624ccf3f654407c1b4badf7d8f9295dd3dabdef65b27677e000744578697420426f6f7420536572766963657320496e766f636174696f6e0100000007000080c00a2e01c85deae718a530ad8c6d20a84009babe6c8989269e950d8cf440c6e997695e64d455c4174a652cd080f6230b7400a04578697420426f6f742053657276696365732052657475726e6564207769746820537563636573730300000001000008c0f9974020ef507068183313d0ca808e0d1ca9b2d1ad0c61f5784e7157c362c06536f5ddacdad4451693f48fcc72fff6244073797374656d2d707265706172696e67000300000001000008c028270906ac86407f4cd8db92605c4c31e8c291362197fd41a456a8f20207f7028060af32be4bc3882e4ac0989f91a496186170702d69645060618cd0c23d15d921b788b8fc3a0d024ac722950300000001000008c0d7335445a782543380dbc9542488789ac2131a1b061135597923895b8eb1c99dcd1e0c5a4ecae0763564716225b2457530636f6d706f73652d686173688060618cd0c23d15d921b788b8fc3a0d024ac72295185da3ba41d5a065bc7aa8330300000001000008c0aa25b837d81098d56e726cdfd4664eae9a1269362479753d04bdfd239a45388a056f245841910931ea09ff8278b9d4823c6770752d706f6c6963792d686173688044136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a0300000001000008c0b9605b83485eccc2609d197bb0c2956dffbb9ae12c06c1c954e7f7625e780e1f6bf7f1e7716da84d6ec641ec4b6da3132c696e7374616e63652d696450d1d54fbba5d0ae0c22b02b126e28d448212e245c0300000001000008c098bd7e6bd3952720b65027fd494834045d06b4a714bf737a06b874638b3ea00ff402f7f583e3e3b05e921c8570433ac630626f6f742d6d722d646f6e65000300000001000008c07f788209b6106d43ad4b0cef85797120c234f175a10b7cb609a73ec1f4285adfe7abca2e1e3d3bd2ba058c2b78d1771d306b65792d70726f76696465725c7b226e616d65223a226e6f6e65222c226964223a22227d0300000001000008c0ba51104636900268b0e059fa3d266419d079d1e94aea26fb9fcbb8d764bf4c89a67ac271b8a0d1a3989945132a111fc72873746f726167652d66730c7a66730300000001000008c01a76b2a80a0be71eae59f80945d876351a7a3fb8e9fd1ff1cede5734aa84ea11fd72b4edfbb6f04e5a85edd114c751bd3073797374656d2d726561647900244073797374656d2d707265706172696e6700186170702d69645060618cd0c23d15d921b788b8fc3a0d024ac7229530636f6d706f73652d686173688060618cd0c23d15d921b788b8fc3a0d024ac72295185da3ba41d5a065bc7aa8333c6770752d706f6c6963792d686173688044136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a2c696e7374616e63652d696450d1d54fbba5d0ae0c22b02b126e28d448212e245c30626f6f742d6d722d646f6e6500306b65792d70726f76696465725c7b226e616d65223a226e6f6e65222c226964223a22227d2873746f726167652d66730c7a66733073797374656d2d7265616479004242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424239237b226f735f696d6167655f68617368223a2231343430626266356233613739626365663462623630313364383264353263313264353366653864313963323163363566623662363336383635313739636131222c226370755f636f756e74223a322c226d656d6f72795f73697a65223a323134373438333634382c2271656d755f76657273696f6e223a22382e322e32222c227063695f686f6c6536345f73697a65223a302c22687567657061676573223a66616c73652c226e756d5f67707573223a302c226e756d5f6e767377697463686573223a302c22686f74706c75675f6f6666223a66616c73652c22696d616765223a2264737461636b2d302e362e302d7263302d7634222c22686f73745f73686172655f6d6f6465223a223970222c226f766d665f76617269616e74223a22707265323032353035222c227464785f6174746573746174696f6e5f76617269616e74223a226c697465222c227464785f6d6561737572656d656e74223a7b22636865636b73756d5f66696c65223a225a5456695a6a646c4d5463344d574d354d5745315a546b304e6a41355a475a6b4d6d5269593249344f54566b5a475a6d5a4759354d6a4d774e32517a4d47466d5a6a6b334e475133596a5135595441794f474d305953416762335a745a69356d5a416f354d324e6d4f544a684e5445314d44677a595752685a47466a4d54557a5954526a4e4463784e54426a595459774f475a69595745344e7a646d4e3245315a574e6d4e6d45774e6d4e694e324e685a444d7a4e6d493449434269656b6c745957646c436d55795a44646a4d6d5a684d6d51774f5441354e47466d4d4745795a446732596a51315954566b5a44526a4d6a63785a54466d4d474d314d4464684e4441335954417a597a646b5a54417859325a685a544d324e47496749476c75615852795957316d6379356a63476c764c6d6436436d466d4d6a52684e324a684d4755304e47466a4e6a64694e4446694d7a52684e6a4e695a4464684d5459784d574a695a54466c4d5759795a6a466b595452694e7a4d775a546b30597a51304d7a633059575a6c4f4451674947316c6447466b595852684c6d707a6232344b4d4445324d474a6b4f574d78596d45775a6d59334e6a637a4d544d34596d4e6d4d474a695a4467314f475530595745784e7a41304d7a55774f475a6d5a6d4e694e4455304d546c6d4d6a637a4e544d7759544d305953416762575668633356795a57316c626e5175644752344c6d4e696233494b5a4459304d6d457a4e4745345a6a41304f4755794e6d5a6b5a546b774f544a6a4e444d314d4451335a6a63795a6d4a695a4442694d7a4d324e444d354d4467794f5745784e4455314d4441334e546335596d45305a43416762575668633356795a57316c626e5175633235774c6d4e696233494b4d544e6c5a444d775932566a5954453559324a694d575a6d596a41334f544668595449305a6d4d785a6d466a595463344e5445314d575669595441315a4751325a4451324e446778597a6c6d5a4455324e324a685a69416762575668633356795a57316c626e51755a324e774c6d4e696233494b4f574a6d4e4455794d474d774e7a466b5a4759314d3252685a4463305a574531597a49334d5468685a6d55344d54426d595455784d5451314d4755315a6d55344e325131597a6c684d4445324d544e6a4e6d4e6d4f53416762575668633356795a57316c626e51755958647a4c6d4e696233494b5a6d5668597a526a596a497a4e4452684d544a6c5957526b4f474935596a45325a544669596a457a4d7a59344e3256684d6a5a684e474d774d545a6b5a5745344d444178597a5a6a4f446b325a446b35595442694f53416762575668633356795a57316c626e51755958647a4c6e4a6c63477868655335716332397543673d3d222c226d6561737572656d656e74223a226f3264325a584a7a61573975424756706257466e5a614e6e5932316b62476c755a586b4248324e76626e4e766247553964485235557a41676157357064443076615735706443427759573570597a30784947356c644335705a6d35686257567a50544167596d6c766332526c646d3568625755394d4342745932553962325a6d4947397663484d396347467561574d6763474e70505735765a57467962486b6763474e70505735766257316a6232356d49484a68626d527662533530636e567a6446396a63485539655342795957356b6232307564484a3163335266596d3976644778765957526c636a31754948527a597a31795a57787059574a735a53427562793172646d316a6247396a6179426b6333526859327375636d397664475a7a58326868633267394e4451784e575133596d566a59324e68596d45314e44597a4f5449794f5745774e6a4d795a5745304e32597a593259324d324a6a4f475a694e54686d5a54457a4e57557a4d474d784d546c6d5a6d45774f54466c4e43426b6333526859327375636d397664475a7a58334e70656d55394e5467354d7a59784d5455796332746c636d356c624639686458526f5a57353061574e765a4756594d4b6263556564466f4f55337239333866675a5763516c704c6e684138444e5041354b762f714e514f64454c7530594e6873637844494c6567436d684559467538473170626d6c30636d5266633268684d7a67305744424f4e507a71446c61386b31677273367a5174386e74657a7930476f785a676d327470514d2b4c2f50487a6374374e47564330444e6b4a59464654614a7962734e6b644752325a714e6b62335a745a6d6c77636d55794d4449314d44566b62584a305a4b4a7263326c755a32786c5833426863334e594d4966536b61726a7055314d484b65457642706f3043383635777837782b7868754e79705870467038536b443139566357572b6f775a714a594476754247374f66326830643239666347467a6331677738674d744e3953433836716a74494c6e33347a5a462b754751646d645562713254503975564a3056395936303348386763325839516b71456b317835754d74305a6e526b58326876596b794145416b454141594a416773434542413d227d2c22737065635f76657273696f6e223a317d" } diff --git a/dstack/verifier/fixtures/tdx-lite-getquote.json b/dstack/verifier/fixtures/tdx-lite-getquote.json index 57a30c837..c5f035600 100644 --- a/dstack/verifier/fixtures/tdx-lite-getquote.json +++ b/dstack/verifier/fixtures/tdx-lite-getquote.json @@ -1,6 +1,6 @@ { - "quote": "040002008100000000000000939a7233f79c4ca9940a0db3957f06071026ff2bbebac59cc1ef911279d9481b000000000c010400000000000000000000000000d0d80c085166ba78ccc69af268e5753cf0f3394523cb4ff7c50b08d9265c82489c099c377be6a400e4d2b57da924012c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000e702060000000000fd685522ce791dfef67414614eb07d03fc07a32c5a66f36288b329dab92b724b1564c73d436ffb9ea84488c51ac5a1c50186b0e55f2fa8e4fb69d890f14f54d5612707646e2573d54e0d2ddaaade77caa9000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f8438db36b96f85d8752ff7f24a89ec05c79ec9eda2ba732c897fb970ca429365b7471b1c054cb84f17b1c2b23ba66402023546e7f3b9d1228e274f70c44d481162540f8452544520a796a52f06879709b81a824a26792a7822327504b0d2aee4c1b739ed451a637b0f82642e48a5ea83925d23633c72e7385c8e9aca4175e133ed1625b7d92eb39edf509c27ff392dc6f24c170d0fd63fc2b1b53202eea47b013978437fa6982cf5e0438ff95c208994aaa0f4ebab2e3a66824b5b56869137e646970313a3a736563703235366b31632d706b3a41353570576d74654a494a4f6a385f7049372d707a654478793147327131384744763838484e526442586b51cc1000008bca152d0454bdfd5adab1bc3a527884f77ea7993d32ee0e4426b2ae0fe42bf3f5642d6abd763b4f4c6042133e2ed79cce743f2c54ff4c7ea5d712dc1172ec244fe5b32ac6ffeb104614bcb8894c7aaafbbbe6f6bfd852f5dcd6cf400557ee764e62850d955975d93eff63b17e6e13e329a7bb13926706c0430017d543ab01920600461000000404191b04ff0006000000000000000000000000000000000000000000000000000000000000000000000000000000001500000000000000e700000000000000e5a3a7b5d830c2953b98534c6c59a3a34fdc34e933f7f5898f0a85cf08846bca0000000000000000000000000000000000000000000000000000000000000000dc9e2a7c6f948f17474e34a7fc43ed030f7c1563f1babddf6340c82e0e54a8c500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000600000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c1140324365c08f021a721dbe9175cb89dcd2235e2bd00bfb235b2a66b8c783600000000000000000000000000000000000000000000000000000000000000002af8cd12d44e0d22f904b15c02968b57b668e7f2487ba308e1d9a269ea125e48b243f7d32bb8551e1e3c2c09bd2162d36941eeb47be50b9b55a766a14d0cfe302000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f05005e0e00002d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d49494538444343424a6167417749424167495556706163774c766c316d476155506b384b4375504141334769465177436759494b6f5a497a6a3045417749770a634445694d434147413155454177775a535735305a577767553064594946424453794251624746305a6d397962534244515445614d42674741315545436777520a535735305a577767513239796347397959585270623234784644415342674e564241634d43314e68626e526849454e7359584a684d51737743515944565151490a44414a445154454c4d416b474131554542684d4356564d774868634e4d6a59774e4445314d4441314d4455345768634e4d7a4d774e4445314d4441314d4455340a576a42774d534977494159445651514444426c4a626e526c624342545231676755454e4c49454e6c636e52705a6d6c6a5958526c4d526f77474159445651514b0a4442464a626e526c6243424462334a7762334a6864476c76626a45554d424947413155454277774c553246756447456751327868636d4578437a414a42674e560a4241674d416b4e424d517377435159445651514745774a56557a425a4d424d4742797147534d34394167454743437147534d343941774548413049414245586a0a53374265726c3262726b65543677707878436a556536564775577268586e51767a41395862524768356b68637671766b566b427874715935475759544f6551340a5948496a636b7974734c6c5531774b594a74576a67674d4d4d4949444344416642674e5648534d4547444157674253566231334e765276683655424a796454300a4d383442567776655644427242674e56485238455a4442694d47436758714263686c706f64485277637a6f764c32467761533530636e567a6447566b633256790a646d6c6a5a584d75615735305a577775593239744c334e6e6543396a5a584a3061575a7059324630615739754c3359304c33426a61324e796244396a595431770a624746305a6d397962535a6c626d4e765a476c755a7a316b5a584977485159445652304f42425945464362386b6b73714d364c384f6765734c713943337339440a7a5333504d41344741315564447745422f775145417749477744414d42674e5648524d4241663845416a41414d4949434f51594a4b6f5a496876684e415130420a424949434b6a4343416959774867594b4b6f5a496876684e4151304241515151514e367178312b487a7758704c373859496b716c646a434341574d47436971470a534962345451454e41514977676746544d42414743797147534962345451454e41514942416745454d42414743797147534962345451454e41514943416745450a4d42414743797147534962345451454e41514944416745434d42414743797147534962345451454e41514945416745434d42414743797147534962345451454e0a41514946416745454d42414743797147534962345451454e41514947416745424d42414743797147534962345451454e41514948416745414d424147437971470a534962345451454e41514949416745464d42414743797147534962345451454e4151494a416745414d42414743797147534962345451454e4151494b416745410a4d42414743797147534962345451454e4151494c416745414d42414743797147534962345451454e4151494d416745414d42414743797147534962345451454e0a4151494e416745414d42414743797147534962345451454e4151494f416745414d42414743797147534962345451454e41514950416745414d424147437971470a534962345451454e41514951416745414d42414743797147534962345451454e415149524167454e4d42384743797147534962345451454e41514953424241450a42414943424145414251414141414141414141414d42414743697147534962345451454e41514d45416741414d42514743697147534962345451454e415151450a42704441627741414144415042676f71686b69472b45304244514546436745424d42344743697147534962345451454e4151594545464a37386f7137314543670a6c7536335265417a675430775241594b4b6f5a496876684e41513042427a41324d42414743797147534962345451454e415163424151482f4d424147437971470a534962345451454e41516343415145414d42414743797147534962345451454e415163444151482f4d416f4743437147534d343942414d43413067414d4555430a494778676472434e7a344753716d32647a4c45533874757663717230444d692b427537533771537133325343416945417439454f6377584f6a31484a4c4462750a6d473357414549577962624f61635959612b7253384366526c514d3d0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436c6a4343416a32674177494241674956414a567658633239472b487051456e4a3150517a7a674658433935554d416f4743437147534d343942414d430a4d476778476a415942674e5642414d4d45556c756447567349464e48574342536232393049454e424d526f77474159445651514b4442464a626e526c624342440a62334a7762334a6864476c76626a45554d424947413155454277774c553246756447456751327868636d4578437a414a42674e564241674d416b4e424d5173770a435159445651514745774a56557a4165467730784f4441314d6a45784d4455774d5442614677307a4d7a41314d6a45784d4455774d5442614d484178496a41670a42674e5642414d4d47556c756447567349464e4857434251513073675547786864475a76636d306751304578476a415942674e5642416f4d45556c75644756730a49454e76636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b474131554543417743513045780a437a414a42674e5642415954416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a304441516344516741454e53422f377432316c58534f0a3243757a7078773734654a423732457944476757357258437478327456544c7136684b6b367a2b5569525a436e71523770734f766771466553786c6d546c4a6c0a65546d693257597a33714f42757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f536347724442530a42674e5648523845537a424a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b633256790a646d6c6a5a584d75615735305a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e5648513445466751556c5739640a7a62306234656c4153636e553944504f4156634c336c517744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159420a4166384341514177436759494b6f5a497a6a30454177494452774177524149675873566b6930772b6936565947573355462f32327561586530594a446a3155650a6e412b546a44316169356343494359623153416d4435786b66545670766f34556f79695359787244574c6d5552344349394e4b7966504e2b0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436a7a4343416a53674177494241674955496d554d316c71644e496e7a6737535655723951477a6b6e42717777436759494b6f5a497a6a3045417749770a614445614d4267474131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e760a636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a0a42674e5642415954416c56544d423458445445344d4455794d5445774e4455784d466f58445451354d54497a4d54497a4e546b314f566f77614445614d4267470a4131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e76636e4276636d46300a615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a42674e56424159540a416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a3044415163445167414543366e45774d4449595a4f6a2f69505773437a61454b69370a314f694f534c52466857476a626e42564a66566e6b59347533496a6b4459594c304d784f346d717379596a6c42616c54565978465032734a424b357a6c4b4f420a757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f5363477244425342674e5648523845537a424a0a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b63325679646d6c6a5a584d75615735300a5a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e564851344546675155496d554d316c71644e496e7a673753560a55723951477a6b6e4271777744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159424166384341514577436759490a4b6f5a497a6a3045417749445351417752674968414f572f35516b522b533943695344634e6f6f774c7550524c735747662f59693747535839344267775477670a41694541344a306c72486f4d732b586f356f2f7358364f39515778485241765a55474f6452513763767152586171493d0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "event_log": "[{\"imr\":0,\"event_type\":2147483659,\"digest\":\"0b8772e5b0b41b83e6044a68397e02f49fb47066b4fbe4917ea2c45c64f323fdacbb37948f821ebaf8bc9c938ba8a749\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483658,\"digest\":\"344bc51c980ba621aaa00da3ed7436f7d6e549197dfe699515dfa2c6583d95e6412af21c097d473155875ffd561d6790\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483649,\"digest\":\"9dc3a1f80bcec915391dcda5ffbb15e7419f77eab462bbf72b42166fb70d50325e37b36f93537a863769bcf9bedae6fb\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483649,\"digest\":\"6f2e3cbc14f9def86980f5f66fd85e99d63e69a73014ed8a5633ce56eca5b64b692108c56110e22acadcef58c3250f1b\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483649,\"digest\":\"d607c0efb41c0d757d69bca0615c3a9ac0b1db06c557d992e906c6b7dee40e0e031640c7bfd7bcd35844ef9edeadc6f9\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483649,\"digest\":\"08a74f8963b337acb6c93682f934496373679dd26af1089cb4eaf0c30cf260a12e814856385ab8843e56a9acea19e127\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483649,\"digest\":\"18cc6e01f0c6ea99aa23f8a280423e94ad81d96d0aeb5180504fc0f7a40cb3619dd39bd6a95ec1680a86ed6ab0f9828d\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":4,\"digest\":\"394341b7182cd227c5c6b07ef8000cdfd86136c4292b8e576573ad7ed9ae41019f5818b4b971c9effc60e1ad9f1289f0\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":10,\"digest\":\"095d04cf26fe03aef6e3561fa24c1aa1cea93f4aeaf563b1f9f7616184c53454875925759434769cec2490acb563a337\",\"event\":\"acpi-loader\",\"event_payload\":\"414350492044415441\"},{\"imr\":0,\"event_type\":10,\"digest\":\"8d9a4d4777a1bc77ecd9d8d37a4628129a80052a510320159a20a923bd07a0e90d8d1f2e1ebf088992b25f0d0fa672ef\",\"event\":\"acpi-rsdp\",\"event_payload\":\"414350492044415441\"},{\"imr\":0,\"event_type\":10,\"digest\":\"3070721e169bc41884724cb0e6b3082e1baf249083d8b389181ba50b9afa951057876c380b8870e8c2facf2eff67a2b6\",\"event\":\"acpi-tables\",\"event_payload\":\"414350492044415441\"},{\"imr\":1,\"event_type\":2147483651,\"digest\":\"ac7e632dcf5cd2a1fe5c1f41f4d9b8219570e64ed3c61038fdbf25404e6f542ffd57f276bc5076307efaf882e6d64177\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483650,\"digest\":\"1dd6f7b457ad880d840d41c961283bab688e94e4b59359ea45686581e90feccea3c624b1226113f824f315eb60ae0a7c\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483650,\"digest\":\"23ada07f5261f12f34a0bd8e46760962d6b4d576a416f1fea1c64bc656b1d28eacf7047ae6e967c58fd2a98bfa74c298\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":1,\"event_type\":2147483655,\"digest\":\"77a0dab2312b4e1e57a84d865a21e5b2ee8d677a21012ada819d0a98988078d3d740f6346bfe0abaa938ca20439a8d71\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":1,\"event_type\":4,\"digest\":\"394341b7182cd227c5c6b07ef8000cdfd86136c4292b8e576573ad7ed9ae41019f5818b4b971c9effc60e1ad9f1289f0\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":2,\"event_type\":6,\"digest\":\"786280842b7364287a3a70d96f7e309252857beb45fb1f91314a2ea863db0adc04c8431ecbf29a966405604631a5aab8\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":2,\"event_type\":6,\"digest\":\"4fe4f7710134a61d7def357add6ac50bdbfeee5032a4c100375e207216ffe42a3bd5822b24e679f91501fff795b81521\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":1,\"event_type\":2147483655,\"digest\":\"214b0bef1379756011344877743fdc2a5382bac6e70362d624ccf3f654407c1b4badf7d8f9295dd3dabdef65b27677e0\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":1,\"event_type\":2147483655,\"digest\":\"0a2e01c85deae718a530ad8c6d20a84009babe6c8989269e950d8cf440c6e997695e64d455c4174a652cd080f6230b74\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"\",\"event\":\"system-preparing\",\"event_payload\":\"\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"\",\"event\":\"app-id\",\"event_payload\":\"86b0e55f2fa8e4fb69d890f14f54d5612707646e\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"\",\"event\":\"compose-hash\",\"event_payload\":\"86b0e55f2fa8e4fb69d890f14f54d5612707646e2573d54e0d2ddaaade77caa9\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"\",\"event\":\"instance-id\",\"event_payload\":\"050bf89570575fe8fab4cb8f0a62a9e64efe8ead\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"\",\"event\":\"boot-mr-done\",\"event_payload\":\"\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"\",\"event\":\"os-image-hash\",\"event_payload\":\"07a2388c7a6a1b6a646d443f1517990a4ec294471d63146cda9d56972765051d\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"\",\"event\":\"key-provider\",\"event_payload\":\"7b226e616d65223a226b6d73222c226964223a223330353933303133303630373261383634386365336430323031303630383261383634386365336430333031303730333432303030343266373165323334643733333961316365616361303963336333393165623831366335333366393830616461616233346631366561643039336666306163313030643963303332353361333035366636643237373335313235343333313830623365363163353461373866336664313333333738363965303035316465653036227d\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"\",\"event\":\"storage-fs\",\"event_payload\":\"7a6673\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"\",\"event\":\"system-ready\",\"event_payload\":\"\"}]", - "report_data": "646970313a3a736563703235366b31632d706b3a41353570576d74654a494a4f6a385f7049372d707a654478793147327131384744763838484e526442586b51", - "vm_config": "{\"os_image_hash\":\"e6f5cfec20c02e7b97baa213d0f718020b55e040172d90ccbcb946d56c8b09db\",\"cpu_count\":2,\"memory_size\":2147483648,\"qemu_version\":\"8.2.2\",\"pci_hole64_size\":0,\"hugepages\":false,\"num_gpus\":0,\"num_nvswitches\":0,\"hotplug_off\":false,\"image\":\"dstack-0.6.0\",\"host_share_mode\":\"9p\",\"ovmf_variant\":\"pre202505\",\"tdx_attestation_variant\":\"lite\",\"tdx_measurement\":{\"measurement\":\"o2d2ZXJzaW9uA2VpbWFnZaNuY21kbGluZV9zaGEzODRYMHhigIQrc2Qoejpw2W9+MJJShXvrRfsfkTFKLqhj2wrcBMhDHsvympZkBWBGMaWquHNrZXJuZWxfYXV0aGVudGljb2RlWDCsfmMtz1zSof5cH0H02bghlXDmTtPGEDj9vyVATm9UL/1X8na8UHYwfvr4gubWQXdtaW5pdHJkX3NoYTM4NFgwT+T3cQE0ph197zV63WrFC9v+7lAypMEAN14gchb/5Co71YIrJOZ5+RUB//eVuBUhZHRkdmajZG92bWZpcHJlMjAyNTA1ZG1ydGSia3NpbmdsZV9wYXNzWDCm8qyUUYEGhqTbJZ/o+lQ43EpYvan9L1sfsJKDNXBVANKaFckjh0FqL1Ld3Omcg/hodHdvX3Bhc3NYMP1oVSLOeR3+9nQUYU6wfQP8B6MsWmbzYoizKdq5K3JLFWTHPUNv+56oRIjFGsWhxWZ0ZF9ob2JMgBAJBAAGCQILAhAQ\",\"checksum_file\":\"OGM5ZTk1NWYwYjU3N2NjODk0NWE4OTFhMzRjY2ZhNWI4ZTA4YmI1ZTVkMjQ5OTkyNjAzZTY0ZGQxNDg3MGUzYyAgbWVhc3VyZW1lbnQudGR4LmNib3IK\"},\"spec_version\":1}" + "quote": "040002008100000000000000939a7233f79c4ca9940a0db3957f0607dca47981c0d1095de780f491950bc3c9000000000e010400000000000000000000000000346bc77a1846cac214dd2e8edeb9ee4349449d6c3f9ff2c52149a634c27b7fd1bd314c2ef6b973eeebd55742952531a100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000e702060000000000f2032d37d482f3aaa3b482e7df8cd917eb8641d99d51bab64cff6e549d15f58eb4dc7f207365fd424a84935c79b8cb740160618cd0c23d15d921b788b8fc3a0d024ac72295185da3ba41d5a065bc7aa833000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f8438db36b96f85d8752ff7f24a89ec05c79ec9eda2ba732c897fb970ca429365b7471b1c054cb84f17b1c2b23ba66404ffc8bcd33abc1bbf1d772196dc6d9f68cbe64c07b16a74ac085aa22ae976731bbb34a59255f119ec1b2ba53018cfa8eea91d5b853964f711fd60c71d1d4a96ebe975801610a869ebedb95fcaa372d4961377d139a6f3a3b21c4ac4352f70748b49119652a84beeee90c614992c1f67461b17edac4a4f6034c72b147c51fae66920eb2d23e8d40287fcebf5ac2a263e842424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242cc100000c74787e57f2b0bfd851e4d805e82324b8011f475fe9025ca9a58bc59ad5c7aed3daa69e2bb78497d496a8fba19c32d64fcfe5c9cf4808bf0be5236872a1a10c4445e655dfb4499fb6e3eb434c3275c340df9efb896c56d628111c4b955c15bf31774c93727636fdbbc8236633bb83f08388c368684d3c75e29aed47ec050e7720600461000000404191b04ff0006000000000000000000000000000000000000000000000000000000000000000000000000000000001500000000000000e700000000000000e5a3a7b5d830c2953b98534c6c59a3a34fdc34e933f7f5898f0a85cf08846bca0000000000000000000000000000000000000000000000000000000000000000dc9e2a7c6f948f17474e34a7fc43ed030f7c1563f1babddf6340c82e0e54a8c500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000600000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c209022cccc4d771dc1ede10db23ac55be3419d22901a89fb8cc6138404337b20000000000000000000000000000000000000000000000000000000000000000ccdef229483b0f91c5dd1e85e36ec2b9f74ff2707cb97e5ada1f291c3f73597f0022b3cdc2af07c29adec61cf70877f4b9d14f89d2dd63d424a28bd6716b895a2000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f05005e0e00002d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d49494538444343424a65674177494241674956414b576652394d75626c4e5642566e4250454a58614b6466567151394d416f4743437147534d343942414d430a4d484178496a416742674e5642414d4d47556c756447567349464e4857434251513073675547786864475a76636d306751304578476a415942674e5642416f4d0a45556c756447567349454e76636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155450a4341774351304578437a414a42674e5642415954416c56544d423458445449324d4463794e6a41334e4445774f466f5844544d7a4d4463794e6a41334e4445770a4f466f77634445694d434147413155454177775a535735305a5777675530645949464244537942445a584a3061575a70593246305a5445614d426747413155450a43677752535735305a577767513239796347397959585270623234784644415342674e564241634d43314e68626e526849454e7359584a684d517377435159440a5651514944414a445154454c4d416b474131554542684d4356564d775754415442676371686b6a4f5051494242676771686b6a4f50514d4242774e43414151510a644b6d76382f774831684e6476705639474d476345442b514b764f382b6276676f752b426362774c432b79582f78427778384d33395444347262436636764a570a54786f442f496d61784b2b337857554864704d496f3449444444434341776777487759445652306a42426777466f41556c5739647a62306234656c4153636e550a3944504f4156634c336c5177617759445652306642475177596a42676f46366758495a616148523063484d364c79396863476b7564484a316333526c5a484e6c0a636e5a705932567a4c6d6c75644756734c6d4e766253397a5a3367765932567964476c6d61574e6864476c76626939324e4339775932746a636d772f593245390a6347786864475a76636d306d5a57356a62325270626d63395a4756794d42304741315564446751574242515a43536c546e6b372f446b746d702f504a6d304f500a7a417455357a414f42674e56485138424166384542414d434273417744415944565230544151482f4241497741444343416a6b4743537147534962345451454e0a4151534341696f776767496d4d42344743697147534962345451454e415145454547396738357947645143775644587367467967472b6f776767466a42676f710a686b69472b453042445145434d494942557a415142677371686b69472b45304244514543415149424244415142677371686b69472b45304244514543416749420a4244415142677371686b69472b4530424451454341774942416a415142677371686b69472b4530424451454342414942416a415142677371686b69472b4530420a44514543425149424244415142677371686b69472b45304244514543426749424154415142677371686b69472b453042445145434277494241444151426773710a686b69472b45304244514543434149424254415142677371686b69472b45304244514543435149424144415142677371686b69472b45304244514543436749420a4144415142677371686b69472b45304244514543437749424144415142677371686b69472b45304244514543444149424144415142677371686b69472b4530420a44514543445149424144415142677371686b69472b45304244514543446749424144415142677371686b69472b453042445145434477494241444151426773710a686b69472b45304244514543454149424144415142677371686b69472b45304244514543455149424454416642677371686b69472b45304244514543456751510a42415143416751424141554141414141414141414144415142676f71686b69472b45304244514544424149414144415542676f71686b69472b453042445145450a4241615177473841414141774477594b4b6f5a496876684e4151304242516f424154416542676f71686b69472b4530424451454742424238724731754c5a50690a3839584569693555414154684d45514743697147534962345451454e415163774e6a415142677371686b69472b45304244514548415145422f7a4151426773710a686b69472b45304244514548416745422f7a415142677371686b69472b45304244514548417745422f7a414b42676771686b6a4f5051514441674e48414442450a4169414a6d6b77376745482b636d2f4446667275664b3834374f7475546465322f7535615031732f42617237504149675364384c50315161743847506a6a34650a684c2b315470714e64557a2f5946733053515279545941466774343d0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436c6a4343416a32674177494241674956414a567658633239472b487051456e4a3150517a7a674658433935554d416f4743437147534d343942414d430a4d476778476a415942674e5642414d4d45556c756447567349464e48574342536232393049454e424d526f77474159445651514b4442464a626e526c624342440a62334a7762334a6864476c76626a45554d424947413155454277774c553246756447456751327868636d4578437a414a42674e564241674d416b4e424d5173770a435159445651514745774a56557a4165467730784f4441314d6a45784d4455774d5442614677307a4d7a41314d6a45784d4455774d5442614d484178496a41670a42674e5642414d4d47556c756447567349464e4857434251513073675547786864475a76636d306751304578476a415942674e5642416f4d45556c75644756730a49454e76636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b474131554543417743513045780a437a414a42674e5642415954416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a304441516344516741454e53422f377432316c58534f0a3243757a7078773734654a423732457944476757357258437478327456544c7136684b6b367a2b5569525a436e71523770734f766771466553786c6d546c4a6c0a65546d693257597a33714f42757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f536347724442530a42674e5648523845537a424a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b633256790a646d6c6a5a584d75615735305a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e5648513445466751556c5739640a7a62306234656c4153636e553944504f4156634c336c517744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159420a4166384341514177436759494b6f5a497a6a30454177494452774177524149675873566b6930772b6936565947573355462f32327561586530594a446a3155650a6e412b546a44316169356343494359623153416d4435786b66545670766f34556f79695359787244574c6d5552344349394e4b7966504e2b0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436a7a4343416a53674177494241674955496d554d316c71644e496e7a6737535655723951477a6b6e42717777436759494b6f5a497a6a3045417749770a614445614d4267474131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e760a636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a0a42674e5642415954416c56544d423458445445344d4455794d5445774e4455784d466f58445451354d54497a4d54497a4e546b314f566f77614445614d4267470a4131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e76636e4276636d46300a615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a42674e56424159540a416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a3044415163445167414543366e45774d4449595a4f6a2f69505773437a61454b69370a314f694f534c52466857476a626e42564a66566e6b59347533496a6b4459594c304d784f346d717379596a6c42616c54565978465032734a424b357a6c4b4f420a757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f5363477244425342674e5648523845537a424a0a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b63325679646d6c6a5a584d75615735300a5a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e564851344546675155496d554d316c71644e496e7a673753560a55723951477a6b6e4271777744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159424166384341514577436759490a4b6f5a497a6a3045417749445351417752674968414f572f35516b522b533943695344634e6f6f774c7550524c735747662f59693747535839344267775477670a41694541344a306c72486f4d732b586f356f2f7358364f39515778485241765a55474f6452513763767152586171493d0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "event_log": "[{\"imr\":0,\"event_type\":2147483659,\"digest\":\"0b8772e5b0b41b83e6044a68397e02f49fb47066b4fbe4917ea2c45c64f323fdacbb37948f821ebaf8bc9c938ba8a749\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483658,\"digest\":\"344bc51c980ba621aaa00da3ed7436f7d6e549197dfe699515dfa2c6583d95e6412af21c097d473155875ffd561d6790\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483649,\"digest\":\"9dc3a1f80bcec915391dcda5ffbb15e7419f77eab462bbf72b42166fb70d50325e37b36f93537a863769bcf9bedae6fb\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483649,\"digest\":\"6f2e3cbc14f9def86980f5f66fd85e99d63e69a73014ed8a5633ce56eca5b64b692108c56110e22acadcef58c3250f1b\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483649,\"digest\":\"d607c0efb41c0d757d69bca0615c3a9ac0b1db06c557d992e906c6b7dee40e0e031640c7bfd7bcd35844ef9edeadc6f9\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483649,\"digest\":\"08a74f8963b337acb6c93682f934496373679dd26af1089cb4eaf0c30cf260a12e814856385ab8843e56a9acea19e127\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483649,\"digest\":\"18cc6e01f0c6ea99aa23f8a280423e94ad81d96d0aeb5180504fc0f7a40cb3619dd39bd6a95ec1680a86ed6ab0f9828d\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":4,\"digest\":\"394341b7182cd227c5c6b07ef8000cdfd86136c4292b8e576573ad7ed9ae41019f5818b4b971c9effc60e1ad9f1289f0\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":10,\"digest\":\"095d04cf26fe03aef6e3561fa24c1aa1cea93f4aeaf563b1f9f7616184c53454875925759434769cec2490acb563a337\",\"event\":\"acpi-loader\",\"event_payload\":\"414350492044415441\"},{\"imr\":0,\"event_type\":10,\"digest\":\"8d9a4d4777a1bc77ecd9d8d37a4628129a80052a510320159a20a923bd07a0e90d8d1f2e1ebf088992b25f0d0fa672ef\",\"event\":\"acpi-rsdp\",\"event_payload\":\"414350492044415441\"},{\"imr\":0,\"event_type\":10,\"digest\":\"3070721e169bc41884724cb0e6b3082e1baf249083d8b389181ba50b9afa951057876c380b8870e8c2facf2eff67a2b6\",\"event\":\"acpi-tables\",\"event_payload\":\"414350492044415441\"},{\"imr\":1,\"event_type\":2147483651,\"digest\":\"a6dc51e745a0e537afddfc7e06567109692e7840f0334f0392affea35039d10bbb460d86c7310c82de8029a111816ef0\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483650,\"digest\":\"1dd6f7b457ad880d840d41c961283bab688e94e4b59359ea45686581e90feccea3c624b1226113f824f315eb60ae0a7c\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":0,\"event_type\":2147483650,\"digest\":\"23ada07f5261f12f34a0bd8e46760962d6b4d576a416f1fea1c64bc656b1d28eacf7047ae6e967c58fd2a98bfa74c298\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":1,\"event_type\":2147483655,\"digest\":\"77a0dab2312b4e1e57a84d865a21e5b2ee8d677a21012ada819d0a98988078d3d740f6346bfe0abaa938ca20439a8d71\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":1,\"event_type\":4,\"digest\":\"394341b7182cd227c5c6b07ef8000cdfd86136c4292b8e576573ad7ed9ae41019f5818b4b971c9effc60e1ad9f1289f0\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":2,\"event_type\":6,\"digest\":\"fbdcf9ef5e402595d8e2c9c412e3b2839d480d4a145e8347d1ebc8a165d920f55c62221c7f2ac3bfba1b80e5a1f3fc6e\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":2,\"event_type\":6,\"digest\":\"4e34fcea0e56bc93582bb3acd0b7c9ed7b3cb41a8c59826dada5033e2ff3c7cdcb7b346542d033642581454da2726ec3\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":1,\"event_type\":2147483655,\"digest\":\"214b0bef1379756011344877743fdc2a5382bac6e70362d624ccf3f654407c1b4badf7d8f9295dd3dabdef65b27677e0\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":1,\"event_type\":2147483655,\"digest\":\"0a2e01c85deae718a530ad8c6d20a84009babe6c8989269e950d8cf440c6e997695e64d455c4174a652cd080f6230b74\",\"event\":\"\",\"event_payload\":\"\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"f9974020ef507068183313d0ca808e0d1ca9b2d1ad0c61f5784e7157c362c06536f5ddacdad4451693f48fcc72fff624\",\"event\":\"system-preparing\",\"event_payload\":\"\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"28270906ac86407f4cd8db92605c4c31e8c291362197fd41a456a8f20207f7028060af32be4bc3882e4ac0989f91a496\",\"event\":\"app-id\",\"event_payload\":\"60618cd0c23d15d921b788b8fc3a0d024ac72295\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"d7335445a782543380dbc9542488789ac2131a1b061135597923895b8eb1c99dcd1e0c5a4ecae0763564716225b24575\",\"event\":\"compose-hash\",\"event_payload\":\"60618cd0c23d15d921b788b8fc3a0d024ac72295185da3ba41d5a065bc7aa833\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"aa25b837d81098d56e726cdfd4664eae9a1269362479753d04bdfd239a45388a056f245841910931ea09ff8278b9d482\",\"event\":\"gpu-policy-hash\",\"event_payload\":\"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"b9605b83485eccc2609d197bb0c2956dffbb9ae12c06c1c954e7f7625e780e1f6bf7f1e7716da84d6ec641ec4b6da313\",\"event\":\"instance-id\",\"event_payload\":\"d1d54fbba5d0ae0c22b02b126e28d448212e245c\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"98bd7e6bd3952720b65027fd494834045d06b4a714bf737a06b874638b3ea00ff402f7f583e3e3b05e921c8570433ac6\",\"event\":\"boot-mr-done\",\"event_payload\":\"\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"7f788209b6106d43ad4b0cef85797120c234f175a10b7cb609a73ec1f4285adfe7abca2e1e3d3bd2ba058c2b78d1771d\",\"event\":\"key-provider\",\"event_payload\":\"7b226e616d65223a226e6f6e65222c226964223a22227d\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"ba51104636900268b0e059fa3d266419d079d1e94aea26fb9fcbb8d764bf4c89a67ac271b8a0d1a3989945132a111fc7\",\"event\":\"storage-fs\",\"event_payload\":\"7a6673\"},{\"imr\":3,\"event_type\":134217729,\"digest\":\"1a76b2a80a0be71eae59f80945d876351a7a3fb8e9fd1ff1cede5734aa84ea11fd72b4edfbb6f04e5a85edd114c751bd\",\"event\":\"system-ready\",\"event_payload\":\"\"}]", + "report_data": "42424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242", + "vm_config": "{\"os_image_hash\":\"1440bbf5b3a79bcef4bb6013d82d52c12d53fe8d19c21c65fb6b636865179ca1\",\"cpu_count\":2,\"memory_size\":2147483648,\"qemu_version\":\"8.2.2\",\"pci_hole64_size\":0,\"hugepages\":false,\"num_gpus\":0,\"num_nvswitches\":0,\"hotplug_off\":false,\"image\":\"dstack-0.6.0-rc0-v4\",\"host_share_mode\":\"9p\",\"ovmf_variant\":\"pre202505\",\"tdx_attestation_variant\":\"lite\",\"tdx_measurement\":{\"checksum_file\":\"ZTViZjdlMTc4MWM5MWE1ZTk0NjA5ZGZkMmRiY2I4OTVkZGZmZGY5MjMwN2QzMGFmZjk3NGQ3YjQ5YTAyOGM0YSAgb3ZtZi5mZAo5M2NmOTJhNTE1MDgzYWRhZGFjMTUzYTRjNDcxNTBjYTYwOGZiYWE4NzdmN2E1ZWNmNmEwNmNiN2NhZDMzNmI4ICBiekltYWdlCmUyZDdjMmZhMmQwOTA5NGFmMGEyZDg2YjQ1YTVkZDRjMjcxZTFmMGM1MDdhNDA3YTAzYzdkZTAxY2ZhZTM2NGIgIGluaXRyYW1mcy5jcGlvLmd6CmFmMjRhN2JhMGU0NGFjNjdiNDFiMzRhNjNiZDdhMTYxMWJiZTFlMWYyZjFkYTRiNzMwZTk0YzQ0Mzc0YWZlODQgIG1ldGFkYXRhLmpzb24KMDE2MGJkOWMxYmEwZmY3NjczMTM4YmNmMGJiZDg1OGU0YWExNzA0MzUwOGZmZmNiNDU0MTlmMjczNTMwYTM0YSAgbWVhc3VyZW1lbnQudGR4LmNib3IKZDY0MmEzNGE4ZjA0OGUyNmZkZTkwOTJjNDM1MDQ3ZjcyZmJiZDBiMzM2NDM5MDgyOWExNDU1MDA3NTc5YmE0ZCAgbWVhc3VyZW1lbnQuc25wLmNib3IKMTNlZDMwY2VjYTE5Y2JiMWZmYjA3OTFhYTI0ZmMxZmFjYTc4NTE1MWViYTA1ZGQ2ZDQ2NDgxYzlmZDU2N2JhZiAgbWVhc3VyZW1lbnQuZ2NwLmNib3IKOWJmNDUyMGMwNzFkZGY1M2RhZDc0ZWE1YzI3MThhZmU4MTBmYTUxMTQ1MGU1ZmU4N2Q1YzlhMDE2MTNjNmNmOSAgbWVhc3VyZW1lbnQuYXdzLmNib3IKZmVhYzRjYjIzNDRhMTJlYWRkOGI5YjE2ZTFiYjEzMzY4N2VhMjZhNGMwMTZkZWE4MDAxYzZjODk2ZDk5YTBiOSAgbWVhc3VyZW1lbnQuYXdzLnJlcGxheS5qc29uCg==\",\"measurement\":\"o2d2ZXJzaW9uBGVpbWFnZaNnY21kbGluZXkBH2NvbnNvbGU9dHR5UzAgaW5pdD0vaW5pdCBwYW5pYz0xIG5ldC5pZm5hbWVzPTAgYmlvc2Rldm5hbWU9MCBtY2U9b2ZmIG9vcHM9cGFuaWMgcGNpPW5vZWFybHkgcGNpPW5vbW1jb25mIHJhbmRvbS50cnVzdF9jcHU9eSByYW5kb20udHJ1c3RfYm9vdGxvYWRlcj1uIHRzYz1yZWxpYWJsZSBuby1rdm1jbG9jayBkc3RhY2sucm9vdGZzX2hhc2g9NDQxNWQ3YmVjY2NhYmE1NDYzOTIyOWEwNjMyZWE0N2YzY2Y2M2JjOGZiNThmZTEzNWUzMGMxMTlmZmEwOTFlNCBkc3RhY2sucm9vdGZzX3NpemU9NTg5MzYxMTUyc2tlcm5lbF9hdXRoZW50aWNvZGVYMKbcUedFoOU3r938fgZWcQlpLnhA8DNPA5Kv/qNQOdELu0YNhscxDILegCmhEYFu8G1pbml0cmRfc2hhMzg0WDBONPzqDla8k1grs6zQt8ntezy0GoxZgm2tpQM+L/PHzct7NGVC0DNkJYFFTaJybsNkdGR2ZqNkb3ZtZmlwcmUyMDI1MDVkbXJ0ZKJrc2luZ2xlX3Bhc3NYMIfSkarjpU1MHKeEvBpo0C865wx7x+xhuNypXpFp8SkD19VcWW+owZqJYDvuBG7Of2h0d29fcGFzc1gw8gMtN9SC86qjtILn34zZF+uGQdmdUbq2TP9uVJ0V9Y603H8gc2X9QkqEk1x5uMt0ZnRkX2hvYkyAEAkEAAYJAgsCEBA=\"},\"spec_version\":1}" } diff --git a/dstack/verifier/fixtures/tdx-lite-normalized-attestation.json b/dstack/verifier/fixtures/tdx-lite-normalized-attestation.json new file mode 100644 index 000000000..c960b5d2c --- /dev/null +++ b/dstack/verifier/fixtures/tdx-lite-normalized-attestation.json @@ -0,0 +1,3 @@ +{ + "attestation": "0000394e040002008100000000000000939a7233f79c4ca9940a0db3957f0607dca47981c0d1095de780f491950bc3c9000000000e010400000000000000000000000000346bc77a1846cac214dd2e8edeb9ee4349449d6c3f9ff2c52149a634c27b7fd1bd314c2ef6b973eeebd55742952531a100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000e70206000000000078cb3ad79ad26d3217e3d28a42b1a55bab034dfc3478bc910596d36c8f46f17eaf8bbe19f27e9858d01af63d316606fb00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000068102e7b524af310f7b7d426ce75481e36c40f5d513a9009c046e9d37e31551f0134d954b496a3357fd61d03f07ffe9660db95c7f164464e5bb3cc38610c23ade2c81b651256f9712ae55aa4c51ca092d0b550999407de966f35cb878f79d200f873ce9b6d655985280d538d1da3113b7a5b6335a4d63ba534a21a644dbd89528ea7df4f7b174f2bde1d38bb05793535c76695ac02680e1ae25feea17719e206bea64e59e9a209196e38999f70ba5a220a75e433d85522892219e3cd8a92a9c311223344556677880000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000cc1000006d355e57a33e5e2f51368270424c90b3385e6ec13c5aa278d33e6c8f50cb59f2bd3f882f4bd44d2440328204116b96daadfcdbe83d37648db84f122c0da25374445e655dfb4499fb6e3eb434c3275c340df9efb896c56d628111c4b955c15bf31774c93727636fdbbc8236633bb83f08388c368684d3c75e29aed47ec050e7720600461000000404191b04ff0006000000000000000000000000000000000000000000000000000000000000000000000000000000001500000000000000e700000000000000e5a3a7b5d830c2953b98534c6c59a3a34fdc34e933f7f5898f0a85cf08846bca0000000000000000000000000000000000000000000000000000000000000000dc9e2a7c6f948f17474e34a7fc43ed030f7c1563f1babddf6340c82e0e54a8c500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000600000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c209022cccc4d771dc1ede10db23ac55be3419d22901a89fb8cc6138404337b2000000000000000000000000000000000000000000000000000000000000000018264bac729410b741866c05234532ec0ba2725d1a4b6328a73151cdd0681b4c395ea0fb11bca5e540cfa36967d094400ff61b54d12df313517bd02c88e15ab42000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f05005e0e00002d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d49494538444343424a65674177494241674956414b576652394d75626c4e5642566e4250454a58614b6466567151394d416f4743437147534d343942414d430a4d484178496a416742674e5642414d4d47556c756447567349464e4857434251513073675547786864475a76636d306751304578476a415942674e5642416f4d0a45556c756447567349454e76636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155450a4341774351304578437a414a42674e5642415954416c56544d423458445449324d4463794e6a41334e4445774f466f5844544d7a4d4463794e6a41334e4445770a4f466f77634445694d434147413155454177775a535735305a5777675530645949464244537942445a584a3061575a70593246305a5445614d426747413155450a43677752535735305a577767513239796347397959585270623234784644415342674e564241634d43314e68626e526849454e7359584a684d517377435159440a5651514944414a445154454c4d416b474131554542684d4356564d775754415442676371686b6a4f5051494242676771686b6a4f50514d4242774e43414151510a644b6d76382f774831684e6476705639474d476345442b514b764f382b6276676f752b426362774c432b79582f78427778384d33395444347262436636764a570a54786f442f496d61784b2b337857554864704d496f3449444444434341776777487759445652306a42426777466f41556c5739647a62306234656c4153636e550a3944504f4156634c336c5177617759445652306642475177596a42676f46366758495a616148523063484d364c79396863476b7564484a316333526c5a484e6c0a636e5a705932567a4c6d6c75644756734c6d4e766253397a5a3367765932567964476c6d61574e6864476c76626939324e4339775932746a636d772f593245390a6347786864475a76636d306d5a57356a62325270626d63395a4756794d42304741315564446751574242515a43536c546e6b372f446b746d702f504a6d304f500a7a417455357a414f42674e56485138424166384542414d434273417744415944565230544151482f4241497741444343416a6b4743537147534962345451454e0a4151534341696f776767496d4d42344743697147534962345451454e415145454547396738357947645143775644587367467967472b6f776767466a42676f710a686b69472b453042445145434d494942557a415142677371686b69472b45304244514543415149424244415142677371686b69472b45304244514543416749420a4244415142677371686b69472b4530424451454341774942416a415142677371686b69472b4530424451454342414942416a415142677371686b69472b4530420a44514543425149424244415142677371686b69472b45304244514543426749424154415142677371686b69472b453042445145434277494241444151426773710a686b69472b45304244514543434149424254415142677371686b69472b45304244514543435149424144415142677371686b69472b45304244514543436749420a4144415142677371686b69472b45304244514543437749424144415142677371686b69472b45304244514543444149424144415142677371686b69472b4530420a44514543445149424144415142677371686b69472b45304244514543446749424144415142677371686b69472b453042445145434477494241444151426773710a686b69472b45304244514543454149424144415142677371686b69472b45304244514543455149424454416642677371686b69472b45304244514543456751510a42415143416751424141554141414141414141414144415142676f71686b69472b45304244514544424149414144415542676f71686b69472b453042445145450a4241615177473841414141774477594b4b6f5a496876684e4151304242516f424154416542676f71686b69472b4530424451454742424238724731754c5a50690a3839584569693555414154684d45514743697147534962345451454e415163774e6a415142677371686b69472b45304244514548415145422f7a4151426773710a686b69472b45304244514548416745422f7a415142677371686b69472b45304244514548417745422f7a414b42676771686b6a4f5051514441674e48414442450a4169414a6d6b77376745482b636d2f4446667275664b3834374f7475546465322f7535615031732f42617237504149675364384c50315161743847506a6a34650a684c2b315470714e64557a2f5946733053515279545941466774343d0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436c6a4343416a32674177494241674956414a567658633239472b487051456e4a3150517a7a674658433935554d416f4743437147534d343942414d430a4d476778476a415942674e5642414d4d45556c756447567349464e48574342536232393049454e424d526f77474159445651514b4442464a626e526c624342440a62334a7762334a6864476c76626a45554d424947413155454277774c553246756447456751327868636d4578437a414a42674e564241674d416b4e424d5173770a435159445651514745774a56557a4165467730784f4441314d6a45784d4455774d5442614677307a4d7a41314d6a45784d4455774d5442614d484178496a41670a42674e5642414d4d47556c756447567349464e4857434251513073675547786864475a76636d306751304578476a415942674e5642416f4d45556c75644756730a49454e76636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b474131554543417743513045780a437a414a42674e5642415954416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a304441516344516741454e53422f377432316c58534f0a3243757a7078773734654a423732457944476757357258437478327456544c7136684b6b367a2b5569525a436e71523770734f766771466553786c6d546c4a6c0a65546d693257597a33714f42757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f536347724442530a42674e5648523845537a424a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b633256790a646d6c6a5a584d75615735305a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e5648513445466751556c5739640a7a62306234656c4153636e553944504f4156634c336c517744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159420a4166384341514177436759494b6f5a497a6a30454177494452774177524149675873566b6930772b6936565947573355462f32327561586530594a446a3155650a6e412b546a44316169356343494359623153416d4435786b66545670766f34556f79695359787244574c6d5552344349394e4b7966504e2b0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436a7a4343416a53674177494241674955496d554d316c71644e496e7a6737535655723951477a6b6e42717777436759494b6f5a497a6a3045417749770a614445614d4267474131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e760a636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a0a42674e5642415954416c56544d423458445445344d4455794d5445774e4455784d466f58445451354d54497a4d54497a4e546b314f566f77614445614d4267470a4131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e76636e4276636d46300a615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a42674e56424159540a416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a3044415163445167414543366e45774d4449595a4f6a2f69505773437a61454b69370a314f694f534c52466857476a626e42564a66566e6b59347533496a6b4459594c304d784f346d717379596a6c42616c54565978465032734a424b357a6c4b4f420a757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f5363477244425342674e5648523845537a424a0a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b63325679646d6c6a5a584d75615735300a5a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e564851344546675155496d554d316c71644e496e7a673753560a55723951477a6b6e4271777744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159424166384341514577436759490a4b6f5a497a6a3045417749445351417752674968414f572f35516b522b533943695344634e6f6f774c7550524c735747662f59693747535839344267775477670a41694541344a306c72486f4d732b586f356f2f7358364f39515778485241765a55474f6452513763767152586171493d0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002c000000000a000000c039740da2ea1e7e6c69e5e6569755b2ad6d622a8a9983a2c5b32d5574659dbdc4ca8b95b1c766bd610bc5c26dcde842f82c616370692d6c6f6164657224414350492044415441000000000a000000c06385b89d2decb3de678d51faba0ff3506835915818fd3ee53a739aa678cd9c76bdd755adfa0aa714cf35a1abe062b25324616370692d7273647024414350492044415441000000000a000000c005705c3ca9d71e908d5ce70d4331b6aadce5331b9a6a546f222f017fb0e1797b951a3dac72f59cb197a0c897a201ca072c616370692d7461626c6573244143504920444154410300000001000008c0f9974020ef507068183313d0ca808e0d1ca9b2d1ad0c61f5784e7157c362c06536f5ddacdad4451693f48fcc72fff6244073797374656d2d707265706172696e67000300000001000008c0dc64696b724968f9dea1a2c3071d8d52c730876ca45242c2ecfad00c07349dc4bd6c565f5c6bc6f9648260254b35065f186170702d696450217001b5ca75468a42218cf69aba596d0e80e96b0300000001000008c01a2fbce333d59cdd3cd37db0ee22f498d9deb188169631a26fbf40dbdd0a685a8d930cf0dba46bb79bee3a0b868bb1bd30636f6d706f73652d6861736880c97c1e4d612936e2cce80c4d17f65590857394cf50dd7f83f85ea1e4ee47315f0300000001000008c0c924c4d54519c63b6f123f662abe1b4852fa41f3db9dfd9a052324cae2ccab4c96b446974c9c2b6d03dc8e96e3c3f5842c696e7374616e63652d69645032100cea9835087181765994eb3231d9ef7033420300000001000008c098bd7e6bd3952720b65027fd494834045d06b4a714bf737a06b874638b3ea00ff402f7f583e3e3b05e921c8570433ac630626f6f742d6d722d646f6e65000300000001000008c07f788209b6106d43ad4b0cef85797120c234f175a10b7cb609a73ec1f4285adfe7abca2e1e3d3bd2ba058c2b78d1771d306b65792d70726f76696465725c7b226e616d65223a226e6f6e65222c226964223a22227d0300000001000008c0ba51104636900268b0e059fa3d266419d079d1e94aea26fb9fcbb8d764bf4c89a67ac271b8a0d1a3989945132a111fc72873746f726167652d66730c7a66730300000001000008c01a76b2a80a0be71eae59f80945d876351a7a3fb8e9fd1ff1cede5734aa84ea11fd72b4edfbb6f04e5a85edd114c751bd3073797374656d2d726561647900204073797374656d2d707265706172696e6700186170702d696450217001b5ca75468a42218cf69aba596d0e80e96b30636f6d706f73652d6861736880c97c1e4d612936e2cce80c4d17f65590857394cf50dd7f83f85ea1e4ee47315f2c696e7374616e63652d69645032100cea9835087181765994eb3231d9ef70334230626f6f742d6d722d646f6e6500306b65792d70726f76696465725c7b226e616d65223a226e6f6e65222c226964223a22227d2873746f726167652d66730c7a66733073797374656d2d72656164790011223344556677880000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000dd1f7b226f735f696d6167655f68617368223a2261373366396232303363336566386331373536333665313734316638616566303164343236393563613633386233356364316538323539376362656530316437222c226370755f636f756e74223a312c226d656d6f72795f73697a65223a323134373438333634382c2271656d755f76657273696f6e223a22382e322e32222c227063695f686f6c6536345f73697a65223a302c22687567657061676573223a66616c73652c226e756d5f67707573223a302c226e756d5f6e767377697463686573223a302c22686f74706c75675f6f6666223a66616c73652c22696d616765223a2264737461636b2d302e362e30222c22686f73745f73686172655f6d6f6465223a223970222c226f766d665f76617269616e74223a22707265323032353035222c227464785f6174746573746174696f6e5f76617269616e74223a226c697465222c227464785f6d6561737572656d656e74223a7b22636865636b73756d5f66696c65223a224e4445794e6d4a6a5a6a55795a5752694d4755324e7a4d354f4463354e6a4e684d6a49325a6d49325a54686a4e5755785a44526b4f5759785a5467325a545135597a5a69597a67324f4749794f5467344f54566a4e69416762335a745a69356d5a416f7a4d3249794e6d55774f44566d596a566b4f445532597a59354d54557a4d546b784f474e6c4f4759344e325a694d475579596a4d7a4f5451324e5751324e4445344d475978596a4e6d5a4451304d544e6d4e44493149434269656b6c745957646c436d4d325a5759794e6a6330596d55314f5755774d4751335a6a6b774e7a466b5a6a466c4f4449774f4755325a6a45784d32466b4e7a5a684d474a6a596a67305a6a41774f545a684d4467794d54426a4d44557a4d7a4d6749476c75615852795957316d6379356a63476c764c6d6436436a426a596a5979596a4e684e6a68685a5759795a6d526c5a575933596d597a597a51345a4467345a44426a4d5445794f5746684e44466d4e6d49344f574935597a41785a6a4a6d4d474e6a597a4d325a57526c596a67674947316c6447466b595852684c6d707a6232344b596d59794f546779593245774e6a45314e4459794d7a6b31596d49354e7a67345a474d324e6a646b4d6a67304d6a46684d444d305a54426c4e7a63354d446b774f544a6c4f545a68596a55305a44686b4f4755324f53416762575668633356795a57316c626e5175644752344c6d4e696233494b4d6a59314f5751324e44426a4e4759315a5752685a6d49314e4745334d474d344f544e6d596d5a6c5a5459305a5745324e3249774e4745305a6a67335a6a51355a546b304f474d7a59574578597a6c6c4d7a5a6a4f53416762575668633356795a57316c626e5175633235774c6d4e696233494b4d4749784d475a6c4f4755354d574978596a6b354e5467775a47466c4f5752694d544d354e6d46684d4751344e6a63304d6d4d30595751774e6a4a6d4f444131597a4d784d6a6b334e5751354e6a59335a6a4d305a69416762575668633356795a57316c626e51755a324e774c6d4e696233494b222c226d6561737572656d656e74223a226f3264325a584a7a61573975424756706257466e5a61526e5932316b62476c755a586b4248324e76626e4e766247553964485235557a41676157357064443076615735706443427759573570597a30784947356c644335705a6d35686257567a50544167596d6c766332526c646d3568625755394d4342745932553962325a6d4947397663484d396347467561574d6763474e70505735765a57467962486b6763474e70505735766257316a6232356d49484a68626d527662533530636e567a6446396a63485539655342795957356b6232307564484a3163335266596d3976644778765957526c636a31754948527a597a31795a57787059574a735a53427562793172646d316a6247396a6179426b6333526859327375636d397664475a7a58326868633267395a444d794e6d4d334d3245355a4759794e574d79595441304d6d5269595759785a444d345a6a45344f4756684f445578596a6331595749354d575977596a6c6d5a5459344f575a6a4e6a6733597a5a6b4e4455314f53426b6333526859327375636d397664475a7a58334e70656d55394e446b334f4451774d5449346332746c636d356c624639686458526f5a57353061574e765a4756594d49674a5976515535436254774b536b486756766b5878365337556262734a45696c314b386263394e794d6733594b6d642f4a47356462645568447a507379764d48675961325679626d56735832686c5957526c636c397562334a7459577870656d566b39573170626d6c30636d5266633268684d7a673057444250355064784154536d485833764e5872646173554c322f377555444b6b775141335869427946762f6b4b6a76566769736b356e6e354651482f393557344653466b644752325a714e6b62335a745a6d6c77636d55794d4449314d44566b62584a305a4b4a7263326c755a32786c5833426863334e594d493754396a2b4f6f502f5064544d676c376f5773437353485269532f6238535459556a74462b64465471714f50555a334e6278542f63394474573442524e4d31476830643239666347467a63316777654d7336313572536254495834394b4b5172476c5736734454667730654c7952425a625462493947385836766937345a386e3659574e4161396a30785a6762375a6e526b58326876596b794145416b454141594a416773434542413d227d2c22737065635f76657273696f6e223a317d" +} diff --git a/dstack/verifier/fixtures/tdx-lite-normalized-qemu-10-2-attestation.json b/dstack/verifier/fixtures/tdx-lite-normalized-qemu-10-2-attestation.json new file mode 100644 index 000000000..8a8f6e798 --- /dev/null +++ b/dstack/verifier/fixtures/tdx-lite-normalized-qemu-10-2-attestation.json @@ -0,0 +1,3 @@ +{ + "attestation": "0000ed51040002008100000000000000939a7233f79c4ca9940a0db3957f060731891a79b240af169d0029bd8228e6b300000000090304000000000000000000000000002d2de102461684f14c8d0984a09d895e3e9e15944ce020a03b977e1f114d5e1ed32ef666a47fd19a5851b3800eda3afa00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000e7020600000000008ed3f63f8ea0ffcf75332097ba16b02b121d1892fdbf124d8523b45f9d153aaa38f519dcd6f14ff73d0ed5b805134cd4000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000131221a6abfe1dfef41ac2598c1d731b44d19270b39059c59ede8a82ff2c18a812af58cf90d0047085f2bfaf54fb122960db95c7f164464e5bb3cc38610c23ade2c81b651256f9712ae55aa4c51ca092d0b550999407de966f35cb878f79d200f873ce9b6d655985280d538d1da3113b7a5b6335a4d63ba534a21a644dbd89528ea7df4f7b174f2bde1d38bb057935352fe7c9d3c10f0d2e8939f113c3d08e77783a77cdb1aa63c197b530247eead33a3207986a809f1f3026de8755743496a111223344556677880000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000cc1000004b64b94c51e3d107fe508c6101a4a52a33c857a72a54995df3cf9488ccb50e69fbed19c0fb27cf7bfce2a2af949ff9d05b60ac51c03a86716ac5f18068dd7831264b8f688ebefbe63e0dca90ecbae0e5663dd6d49bf6f942256e1a8856ea6f6811166a1aa4ecf88c5d0c727138abc9d00c1c8f852c832f927155bee726e98fb70600461000000406090905ff0002000000000000000000000000000000000000000000000000000000000000000000000000000000001500000000000000e700000000000000de1e1bbbc8edef08283e8ad51ce112b5b11af473050828543a0c73b914a33e710000000000000000000000000000000000000000000000000000000000000000dc9e2a7c6f948f17474e34a7fc43ed030f7c1563f1babddf6340c82e0e54a8c500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000d1f5f8593e4249ffd4c52b1370295e91bed7b991249aeb5fb30d991ebf47b7bf0000000000000000000000000000000000000000000000000000000000000000c0e06e818a55de0997a7c797c217949dcac0a8b82442365cde29f68ed5095b92cc26b41f392b142c3d3a082176d4c685b7769d92633b767a2536653b8ac513e32000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f05005e0e00002d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d49494538544343424a616741774942416749554e3958714f6d566f6a6c424c7856354956514853636f494e41713077436759494b6f5a497a6a3045417749770a634445694d434147413155454177775a535735305a577767553064594946424453794251624746305a6d397962534244515445614d42674741315545436777520a535735305a577767513239796347397959585270623234784644415342674e564241634d43314e68626e526849454e7359584a684d51737743515944565151490a44414a445154454c4d416b474131554542684d4356564d774868634e4d6a59774f5441794d6a41774e6a55325768634e4d7a4d774f5441794d6a41774e6a55320a576a42774d534977494159445651514444426c4a626e526c624342545231676755454e4c49454e6c636e52705a6d6c6a5958526c4d526f77474159445651514b0a4442464a626e526c6243424462334a7762334a6864476c76626a45554d424947413155454277774c553246756447456751327868636d4578437a414a42674e560a4241674d416b4e424d517377435159445651514745774a56557a425a4d424d4742797147534d34394167454743437147534d3439417745484130494142504c6b0a2f7549366b6c4c496b787155332f434b674c5730446a2f6a4f6961443238686d4b6f462b3937473037546f395477707335555a2f44754e555546314658382f6c0a367049427a54734a3244314244547a5a6958536a67674d4d4d4949444344416642674e5648534d4547444157674253566231334e765276683655424a796454300a4d383442567776655644427242674e56485238455a4442694d47436758714263686c706f64485277637a6f764c32467761533530636e567a6447566b633256790a646d6c6a5a584d75615735305a577775593239744c334e6e6543396a5a584a3061575a7059324630615739754c3359304c33426a61324e796244396a595431770a624746305a6d397962535a6c626d4e765a476c755a7a316b5a584977485159445652304f42425945464a72566a75346632416755614d5636526742544a456e370a6e5936394d41344741315564447745422f775145417749477744414d42674e5648524d4241663845416a41414d4949434f51594a4b6f5a496876684e415130420a424949434b6a4343416959774867594b4b6f5a496876684e415130424151515156613032666977465771502f456d6579464939467a44434341574d47436971470a534962345451454e41514977676746544d42414743797147534962345451454e41514942416745454d42414743797147534962345451454e41514943416745450a4d42414743797147534962345451454e41514944416745434d42414743797147534962345451454e41514945416745434d42414743797147534962345451454e0a41514946416745454d42414743797147534962345451454e41514947416745424d42414743797147534962345451454e41514948416745414d424147437971470a534962345451454e41514949416745434d42414743797147534962345451454e4151494a416745414d42414743797147534962345451454e4151494b416745410a4d42414743797147534962345451454e4151494c416745414d42414743797147534962345451454e4151494d416745414d42414743797147534962345451454e0a4151494e416745414d42414743797147534962345451454e4151494f416745414d42414743797147534962345451454e41514950416745414d424147437971470a534962345451454e41514951416745414d42414743797147534962345451454e415149524167454e4d42384743797147534962345451454e41514953424241450a42414943424145414167414141414141414141414d42414743697147534962345451454e41514d45416741414d42514743697147534962345451454e415151450a42674367625167414144415042676f71686b69472b45304244514546436745424d42344743697147534962345451454e41515945454f324e6c4d316f716573740a41326e3249363856664938775241594b4b6f5a496876684e41513042427a41324d42414743797147534962345451454e415163424151482f4d424147437971470a534962345451454e41516343415145414d42414743797147534962345451454e415163444151482f4d416f4743437147534d343942414d4341306b414d4559430a4951435061715878444f6634316655727436444979484132687437524c54676f7662643357423643596e4a3044514968414a35584e5043394f4e7859765049710a6a305a2f394b6442417451584b77514f31374c4430433133597575620a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436c6a4343416a32674177494241674956414a567658633239472b487051456e4a3150517a7a674658433935554d416f4743437147534d343942414d430a4d476778476a415942674e5642414d4d45556c756447567349464e48574342536232393049454e424d526f77474159445651514b4442464a626e526c624342440a62334a7762334a6864476c76626a45554d424947413155454277774c553246756447456751327868636d4578437a414a42674e564241674d416b4e424d5173770a435159445651514745774a56557a4165467730784f4441314d6a45784d4455774d5442614677307a4d7a41314d6a45784d4455774d5442614d484178496a41670a42674e5642414d4d47556c756447567349464e4857434251513073675547786864475a76636d306751304578476a415942674e5642416f4d45556c75644756730a49454e76636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b474131554543417743513045780a437a414a42674e5642415954416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a304441516344516741454e53422f377432316c58534f0a3243757a7078773734654a423732457944476757357258437478327456544c7136684b6b367a2b5569525a436e71523770734f766771466553786c6d546c4a6c0a65546d693257597a33714f42757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f536347724442530a42674e5648523845537a424a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b633256790a646d6c6a5a584d75615735305a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e5648513445466751556c5739640a7a62306234656c4153636e553944504f4156634c336c517744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159420a4166384341514177436759494b6f5a497a6a30454177494452774177524149675873566b6930772b6936565947573355462f32327561586530594a446a3155650a6e412b546a44316169356343494359623153416d4435786b66545670766f34556f79695359787244574c6d5552344349394e4b7966504e2b0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436a7a4343416a53674177494241674955496d554d316c71644e496e7a6737535655723951477a6b6e42717777436759494b6f5a497a6a3045417749770a614445614d4267474131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e760a636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a0a42674e5642415954416c56544d423458445445344d4455794d5445774e4455784d466f58445451354d54497a4d54497a4e546b314f566f77614445614d4267470a4131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e76636e4276636d46300a615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a42674e56424159540a416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a3044415163445167414543366e45774d4449595a4f6a2f69505773437a61454b69370a314f694f534c52466857476a626e42564a66566e6b59347533496a6b4459594c304d784f346d717379596a6c42616c54565978465032734a424b357a6c4b4f420a757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f5363477244425342674e5648523845537a424a0a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b63325679646d6c6a5a584d75615735300a5a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e564851344546675155496d554d316c71644e496e7a673753560a55723951477a6b6e4271777744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159424166384341514577436759490a4b6f5a497a6a3045417749445351417752674968414f572f35516b522b533943695344634e6f6f774c7550524c735747662f59693747535839344267775477670a41694541344a306c72486f4d732b586f356f2f7358364f39515778485241765a55474f6452513763767152586171493d0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002c000000000a000000c0d8c90b5d0f756dcd707c190313b44e3129abc6c20118efdd32b78f84fa64a6c51396fa43debc993dec00b85718bf74912c616370692d6c6f6164657224414350492044415441000000000a000000c0ab0063753f886063d08d1e65031fdcd12e210764b476210edf90e006af701c20df1710ab3fcb7dee651727f4a369f29a24616370692d7273647024414350492044415441000000000a000000c08583382b5eb22b3b7b8194777aa82ae7060c6b5e8387ee59926a9dc429e1069a9215c24dd299cee42611fd1960711c502c616370692d7461626c6573244143504920444154410300000001000008c0f9974020ef507068183313d0ca808e0d1ca9b2d1ad0c61f5784e7157c362c06536f5ddacdad4451693f48fcc72fff6244073797374656d2d707265706172696e67000300000001000008c0dc64696b724968f9dea1a2c3071d8d52c730876ca45242c2ecfad00c07349dc4bd6c565f5c6bc6f9648260254b35065f186170702d696450217001b5ca75468a42218cf69aba596d0e80e96b0300000001000008c01a2fbce333d59cdd3cd37db0ee22f498d9deb188169631a26fbf40dbdd0a685a8d930cf0dba46bb79bee3a0b868bb1bd30636f6d706f73652d6861736880c97c1e4d612936e2cce80c4d17f65590857394cf50dd7f83f85ea1e4ee47315f0300000001000008c0d9420f5364695b9a5003702eb3adeeb09535fd756bbecf3e62a118e600172f90ec20e9d94d442ab4c9d6f703deea28a22c696e7374616e63652d6964500803c9828f9a23eb841996a507c40fe5235179650300000001000008c098bd7e6bd3952720b65027fd494834045d06b4a714bf737a06b874638b3ea00ff402f7f583e3e3b05e921c8570433ac630626f6f742d6d722d646f6e65000300000001000008c07f788209b6106d43ad4b0cef85797120c234f175a10b7cb609a73ec1f4285adfe7abca2e1e3d3bd2ba058c2b78d1771d306b65792d70726f76696465725c7b226e616d65223a226e6f6e65222c226964223a22227d0300000001000008c0ba51104636900268b0e059fa3d266419d079d1e94aea26fb9fcbb8d764bf4c89a67ac271b8a0d1a3989945132a111fc72873746f726167652d66730c7a66730300000001000008c01a76b2a80a0be71eae59f80945d876351a7a3fb8e9fd1ff1cede5734aa84ea11fd72b4edfbb6f04e5a85edd114c751bd3073797374656d2d726561647900204073797374656d2d707265706172696e6700186170702d696450217001b5ca75468a42218cf69aba596d0e80e96b30636f6d706f73652d6861736880c97c1e4d612936e2cce80c4d17f65590857394cf50dd7f83f85ea1e4ee47315f2c696e7374616e63652d6964500803c9828f9a23eb841996a507c40fe52351796530626f6f742d6d722d646f6e6500306b65792d70726f76696465725c7b226e616d65223a226e6f6e65222c226964223a22227d2873746f726167652d66730c7a66733073797374656d2d72656164790011223344556677880000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e11f7b226f735f696d6167655f68617368223a2261373366396232303363336566386331373536333665313734316638616566303164343236393563613633386233356364316538323539376362656530316437222c226370755f636f756e74223a312c226d656d6f72795f73697a65223a323134373438333634382c2271656d755f76657273696f6e223a2231302e322e31222c227063695f686f6c6536345f73697a65223a302c22687567657061676573223a66616c73652c226e756d5f67707573223a302c226e756d5f6e767377697463686573223a302c22686f74706c75675f6f6666223a66616c73652c22696d616765223a2264737461636b2d302e362e30222c22686f73745f73686172655f6d6f6465223a223970222c226f766d665f76617269616e74223a22707265323032353035222c227464785f6174746573746174696f6e5f76617269616e74223a226c697465222c22737065635f76657273696f6e223a312c227464785f6d6561737572656d656e74223a7b22636865636b73756d5f66696c65223a224e4445794e6d4a6a5a6a55795a5752694d4755324e7a4d354f4463354e6a4e684d6a49325a6d49325a54686a4e5755785a44526b4f5759785a5467325a545135597a5a69597a67324f4749794f5467344f54566a4e69416762335a745a69356d5a416f7a4d3249794e6d55774f44566d596a566b4f445532597a59354d54557a4d546b784f474e6c4f4759344e325a694d475579596a4d7a4f5451324e5751324e4445344d475978596a4e6d5a4451304d544e6d4e44493149434269656b6c745957646c436d4d325a5759794e6a6330596d55314f5755774d4751335a6a6b774e7a466b5a6a466c4f4449774f4755325a6a45784d32466b4e7a5a684d474a6a596a67305a6a41774f545a684d4467794d54426a4d44557a4d7a4d6749476c75615852795957316d6379356a63476c764c6d6436436a426a596a5979596a4e684e6a68685a5759795a6d526c5a575933596d597a597a51345a4467345a44426a4d5445794f5746684e44466d4e6d49344f574935597a41785a6a4a6d4d474e6a597a4d325a57526c596a67674947316c6447466b595852684c6d707a6232344b596d59794f546779593245774e6a45314e4459794d7a6b31596d49354e7a67345a474d324e6a646b4d6a67304d6a46684d444d305a54426c4e7a63354d446b774f544a6c4f545a68596a55305a44686b4f4755324f53416762575668633356795a57316c626e5175644752344c6d4e696233494b4d6a59314f5751324e44426a4e4759315a5752685a6d49314e4745334d474d344f544e6d596d5a6c5a5459305a5745324e3249774e4745305a6a67335a6a51355a546b304f474d7a59574578597a6c6c4d7a5a6a4f53416762575668633356795a57316c626e5175633235774c6d4e696233494b4d4749784d475a6c4f4755354d574978596a6b354e5467775a47466c4f5752694d544d354e6d46684d4751344e6a63304d6d4d30595751774e6a4a6d4f444131597a4d784d6a6b334e5751354e6a59335a6a4d305a69416762575668633356795a57316c626e51755a324e774c6d4e696233494b222c226d6561737572656d656e74223a226f3264325a584a7a61573975424756706257466e5a61526e5932316b62476c755a586b4248324e76626e4e766247553964485235557a41676157357064443076615735706443427759573570597a30784947356c644335705a6d35686257567a50544167596d6c766332526c646d3568625755394d4342745932553962325a6d4947397663484d396347467561574d6763474e70505735765a57467962486b6763474e70505735766257316a6232356d49484a68626d527662533530636e567a6446396a63485539655342795957356b6232307564484a3163335266596d3976644778765957526c636a31754948527a597a31795a57787059574a735a53427562793172646d316a6247396a6179426b6333526859327375636d397664475a7a58326868633267395a444d794e6d4d334d3245355a4759794e574d79595441304d6d5269595759785a444d345a6a45344f4756684f445578596a6331595749354d575977596a6c6d5a5459344f575a6a4e6a6733597a5a6b4e4455314f53426b6333526859327375636d397664475a7a58334e70656d55394e446b334f4451774d5449346332746c636d356c624639686458526f5a57353061574e765a4756594d49674a5976515535436254774b536b486756766b5878365337556262734a45696c314b386263394e794d6733594b6d642f4a47356462645568447a507379764d48675961325679626d56735832686c5957526c636c397562334a7459577870656d566b39573170626d6c30636d5266633268684d7a673057444250355064784154536d485833764e5872646173554c322f377555444b6b775141335869427946762f6b4b6a76566769736b356e6e354651482f393557344653466b644752325a714e6b62335a745a6d6c77636d55794d4449314d44566b62584a305a4b4a7263326c755a32786c5833426863334e594d493754396a2b4f6f502f5064544d676c376f5773437353485269532f6238535459556a74462b64465471714f50555a334e6278542f63394474573442524e4d31476830643239666347467a63316777654d7336313572536254495834394b4b5172476c5736734454667730654c7952425a625462493947385836766937345a386e3659574e4161396a30785a6762375a6e526b58326876596b794145416b454141594a416773434542413d227d7d" +} diff --git a/dstack/verifier/fixtures/tdx-lite-v0-attest.json b/dstack/verifier/fixtures/tdx-lite-v0-attest.json new file mode 100644 index 000000000..d9bf8771a --- /dev/null +++ b/dstack/verifier/fixtures/tdx-lite-v0-attest.json @@ -0,0 +1,3 @@ +{ + "attestation": "0000394e040002008100000000000000939a7233f79c4ca9940a0db3957f0607dca47981c0d1095de780f491950bc3c9000000000e010400000000000000000000000000346bc77a1846cac214dd2e8edeb9ee4349449d6c3f9ff2c52149a634c27b7fd1bd314c2ef6b973eeebd55742952531a100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000e702060000000000c757a8fb1e8322b4bfe59fdc4be74f131b00e23806297fbbfa84fd2d6f79a81fb3f9d741bab70f9a084cd7dfd2cdfa1d0153226fcd7b6bc7ea979b0a550bf49cad993f28e2370169a9510e56c90b3029dc000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f8438db36b96f85d8752ff7f24a89ec05c79ec9eda2ba732c897fb970ca429365b7471b1c054cb84f17b1c2b23ba6640448ccad93d7ca89c9187f2d0322352f8ea7156845a2dc213f44a51dcce52090de2e31edec55561e244802c718be918e43452e5f7a96ac1089a848a3159902ec52af39d486380ab5b18565e31b222da4699000808f2ad3a0bc407c2a7baf33ba91a734a4eb4ec4cf573c38aa4e66fd5ef9c81131612fc6ef4b48fab230b86d737e36453a19f40949990999811e5fc206b42424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242cc100000b1868c384e04778f6197e4172de3cecff27a07aa178eb5e72930c2c6259d1af286ae132ac854d594df44911474ad124c9cfa2da9d03d22cdbed7e39174589d95445e655dfb4499fb6e3eb434c3275c340df9efb896c56d628111c4b955c15bf31774c93727636fdbbc8236633bb83f08388c368684d3c75e29aed47ec050e7720600461000000404191b04ff0006000000000000000000000000000000000000000000000000000000000000000000000000000000001500000000000000e700000000000000e5a3a7b5d830c2953b98534c6c59a3a34fdc34e933f7f5898f0a85cf08846bca0000000000000000000000000000000000000000000000000000000000000000dc9e2a7c6f948f17474e34a7fc43ed030f7c1563f1babddf6340c82e0e54a8c500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000600000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c209022cccc4d771dc1ede10db23ac55be3419d22901a89fb8cc6138404337b200000000000000000000000000000000000000000000000000000000000000001967a486aad7e7b4e53bf940040b9e5a1c3edc3a254f4dae0f3d1e6daa5e6d85c5c743c500c2ddee8d89469f67d66049b6d199f0dba027b08e963c1d26608a342000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f05005e0e00002d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d49494538444343424a65674177494241674956414b576652394d75626c4e5642566e4250454a58614b6466567151394d416f4743437147534d343942414d430a4d484178496a416742674e5642414d4d47556c756447567349464e4857434251513073675547786864475a76636d306751304578476a415942674e5642416f4d0a45556c756447567349454e76636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155450a4341774351304578437a414a42674e5642415954416c56544d423458445449324d4463794e6a41334e4445774f466f5844544d7a4d4463794e6a41334e4445770a4f466f77634445694d434147413155454177775a535735305a5777675530645949464244537942445a584a3061575a70593246305a5445614d426747413155450a43677752535735305a577767513239796347397959585270623234784644415342674e564241634d43314e68626e526849454e7359584a684d517377435159440a5651514944414a445154454c4d416b474131554542684d4356564d775754415442676371686b6a4f5051494242676771686b6a4f50514d4242774e43414151510a644b6d76382f774831684e6476705639474d476345442b514b764f382b6276676f752b426362774c432b79582f78427778384d33395444347262436636764a570a54786f442f496d61784b2b337857554864704d496f3449444444434341776777487759445652306a42426777466f41556c5739647a62306234656c4153636e550a3944504f4156634c336c5177617759445652306642475177596a42676f46366758495a616148523063484d364c79396863476b7564484a316333526c5a484e6c0a636e5a705932567a4c6d6c75644756734c6d4e766253397a5a3367765932567964476c6d61574e6864476c76626939324e4339775932746a636d772f593245390a6347786864475a76636d306d5a57356a62325270626d63395a4756794d42304741315564446751574242515a43536c546e6b372f446b746d702f504a6d304f500a7a417455357a414f42674e56485138424166384542414d434273417744415944565230544151482f4241497741444343416a6b4743537147534962345451454e0a4151534341696f776767496d4d42344743697147534962345451454e415145454547396738357947645143775644587367467967472b6f776767466a42676f710a686b69472b453042445145434d494942557a415142677371686b69472b45304244514543415149424244415142677371686b69472b45304244514543416749420a4244415142677371686b69472b4530424451454341774942416a415142677371686b69472b4530424451454342414942416a415142677371686b69472b4530420a44514543425149424244415142677371686b69472b45304244514543426749424154415142677371686b69472b453042445145434277494241444151426773710a686b69472b45304244514543434149424254415142677371686b69472b45304244514543435149424144415142677371686b69472b45304244514543436749420a4144415142677371686b69472b45304244514543437749424144415142677371686b69472b45304244514543444149424144415142677371686b69472b4530420a44514543445149424144415142677371686b69472b45304244514543446749424144415142677371686b69472b453042445145434477494241444151426773710a686b69472b45304244514543454149424144415142677371686b69472b45304244514543455149424454416642677371686b69472b45304244514543456751510a42415143416751424141554141414141414141414144415142676f71686b69472b45304244514544424149414144415542676f71686b69472b453042445145450a4241615177473841414141774477594b4b6f5a496876684e4151304242516f424154416542676f71686b69472b4530424451454742424238724731754c5a50690a3839584569693555414154684d45514743697147534962345451454e415163774e6a415142677371686b69472b45304244514548415145422f7a4151426773710a686b69472b45304244514548416745422f7a415142677371686b69472b45304244514548417745422f7a414b42676771686b6a4f5051514441674e48414442450a4169414a6d6b77376745482b636d2f4446667275664b3834374f7475546465322f7535615031732f42617237504149675364384c50315161743847506a6a34650a684c2b315470714e64557a2f5946733053515279545941466774343d0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436c6a4343416a32674177494241674956414a567658633239472b487051456e4a3150517a7a674658433935554d416f4743437147534d343942414d430a4d476778476a415942674e5642414d4d45556c756447567349464e48574342536232393049454e424d526f77474159445651514b4442464a626e526c624342440a62334a7762334a6864476c76626a45554d424947413155454277774c553246756447456751327868636d4578437a414a42674e564241674d416b4e424d5173770a435159445651514745774a56557a4165467730784f4441314d6a45784d4455774d5442614677307a4d7a41314d6a45784d4455774d5442614d484178496a41670a42674e5642414d4d47556c756447567349464e4857434251513073675547786864475a76636d306751304578476a415942674e5642416f4d45556c75644756730a49454e76636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b474131554543417743513045780a437a414a42674e5642415954416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a304441516344516741454e53422f377432316c58534f0a3243757a7078773734654a423732457944476757357258437478327456544c7136684b6b367a2b5569525a436e71523770734f766771466553786c6d546c4a6c0a65546d693257597a33714f42757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f536347724442530a42674e5648523845537a424a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b633256790a646d6c6a5a584d75615735305a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e5648513445466751556c5739640a7a62306234656c4153636e553944504f4156634c336c517744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159420a4166384341514177436759494b6f5a497a6a30454177494452774177524149675873566b6930772b6936565947573355462f32327561586530594a446a3155650a6e412b546a44316169356343494359623153416d4435786b66545670766f34556f79695359787244574c6d5552344349394e4b7966504e2b0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436a7a4343416a53674177494241674955496d554d316c71644e496e7a6737535655723951477a6b6e42717777436759494b6f5a497a6a3045417749770a614445614d4267474131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e760a636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a0a42674e5642415954416c56544d423458445445344d4455794d5445774e4455784d466f58445451354d54497a4d54497a4e546b314f566f77614445614d4267470a4131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e76636e4276636d46300a615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a42674e56424159540a416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a3044415163445167414543366e45774d4449595a4f6a2f69505773437a61454b69370a314f694f534c52466857476a626e42564a66566e6b59347533496a6b4459594c304d784f346d717379596a6c42616c54565978465032734a424b357a6c4b4f420a757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f5363477244425342674e5648523845537a424a0a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b63325679646d6c6a5a584d75615735300a5a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e564851344546675155496d554d316c71644e496e7a673753560a55723951477a6b6e4271777744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159424166384341514577436759490a4b6f5a497a6a3045417749445351417752674968414f572f35516b522b533943695344634e6f6f774c7550524c735747662f59693747535839344267775477670a41694541344a306c72486f4d732b586f356f2f7358364f39515778485241765a55474f6452513763767152586171493d0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000074000000000b000080c00b8772e5b0b41b83e6044a68397e02f49fb47066b4fbe4917ea2c45c64f323fdacbb37948f821ebaf8bc9c938ba8a74900a8095464785461626c65000100000000000000af96bb93f2b9b84e9462e0ba745642360090800000000000000000000a000080c0344bc51c980ba621aaa00da3ed7436f7d6e549197dfe699515dfa2c6583d95e6412af21c097d473155875ffd561d679000e82946762858585858585858582d585858582d585858582d585858582d58585858585858585858585829000000c0ff0000000000400800000000000000000001000080c09dc3a1f80bcec915391dcda5ffbb15e7419f77eab462bbf72b42166fb70d50325e37b36f93537a863769bcf9bedae6fb00d061dfe48bca93d211aa0d00e098032b8c0a00000000000000000000000000000053006500630075007200650042006f006f0074000000000001000080c06f2e3cbc14f9def86980f5f66fd85e99d63e69a73014ed8a5633ce56eca5b64b692108c56110e22acadcef58c3250f1b009061dfe48bca93d211aa0d00e098032b8c0200000000000000000000000000000050004b000000000001000080c0d607c0efb41c0d757d69bca0615c3a9ac0b1db06c557d992e906c6b7dee40e0e031640c7bfd7bcd35844ef9edeadc6f9009861dfe48bca93d211aa0d00e098032b8c030000000000000000000000000000004b0045004b000000000001000080c008a74f8963b337acb6c93682f934496373679dd26af1089cb4eaf0c30cf260a12e814856385ab8843e56a9acea19e1270090cbb219d73a3d9645a3bcdad00e67656f02000000000000000000000000000000640062000000000001000080c018cc6e01f0c6ea99aa23f8a280423e94ad81d96d0aeb5180504fc0f7a40cb3619dd39bd6a95ec1680a86ed6ab0f9828d0098cbb219d73a3d9645a3bcdad00e67656f030000000000000000000000000000006400620078000000000004000000c0394341b7182cd227c5c6b07ef8000cdfd86136c4292b8e576573ad7ed9ae41019f5818b4b971c9effc60e1ad9f1289f0001000000000000000000a000000c0095d04cf26fe03aef6e3561fa24c1aa1cea93f4aeaf563b1f9f7616184c53454875925759434769cec2490acb563a3372c616370692d6c6f6164657224414350492044415441000000000a000000c08d9a4d4777a1bc77ecd9d8d37a4628129a80052a510320159a20a923bd07a0e90d8d1f2e1ebf088992b25f0d0fa672ef24616370692d7273647024414350492044415441000000000a000000c03070721e169bc41884724cb0e6b3082e1baf249083d8b389181ba50b9afa951057876c380b8870e8c2facf2eff67a2b62c616370692d7461626c6573244143504920444154410100000003000080c0e2b3698b463dd7a2e6a0b10e5a6a302a3b39ce1a179508ab46c7e232dce9c4dd205bf5222a8714b2796c0977ab29d35c00290118008a7a0000000000b2c9000000000000000000000000002a000000000000000403140072f728144ab61e44b8c39ebdd7f893c7040412006b00650072006e0065006c0000007fff04000000000002000080c01dd6f7b457ad880d840d41c961283bab688e94e4b59359ea45686581e90feccea3c624b1226113f824f315eb60ae0a7c00d061dfe48bca93d211aa0d00e098032b8c0900000000000000020000000000000042006f006f0074004f00720064006500720000000000000002000080c023ada07f5261f12f34a0bd8e46760962d6b4d576a416f1fea1c64bc656b1d28eacf7047ae6e967c58fd2a98bfa74c29800b90161dfe48bca93d211aa0d00e098032b8c08000000000000003e0000000000000042006f006f0074003000300030003000090100002c0055006900410070007000000004071400c9bdb87cebf8344faaea3ee4af6516a10406140021aa2c4614760345836e8ab6f46623317fff04000100000007000080c077a0dab2312b4e1e57a84d865a21e5b2ee8d677a21012ada819d0a98988078d3d740f6346bfe0abaa938ca20439a8d7100a043616c6c696e6720454649204170706c69636174696f6e2066726f6d20426f6f74204f7074696f6e0100000004000000c0394341b7182cd227c5c6b07ef8000cdfd86136c4292b8e576573ad7ed9ae41019f5818b4b971c9effc60e1ad9f1289f00010000000000200000006000000c01fb5131405de515fb00205eb87af9ecf7394ccff92ddcfbe7765bcadc0a829712f045aef33eba215efae2c702b4875db0088ed223b8f1a0000004c4f414445445f494d4147453a3a4c6f61644f7074696f6e73000200000006000000c0fc344901ebf77229d70e84f95822e6feedba03961f9409da42eb4721a72a8ed6de2d99b058f16de80befe585d80cdb300054ec223b8f0d0000004c696e757820696e69747264000100000007000080c0214b0bef1379756011344877743fdc2a5382bac6e70362d624ccf3f654407c1b4badf7d8f9295dd3dabdef65b27677e000744578697420426f6f7420536572766963657320496e766f636174696f6e0100000007000080c00a2e01c85deae718a530ad8c6d20a84009babe6c8989269e950d8cf440c6e997695e64d455c4174a652cd080f6230b7400a04578697420426f6f742053657276696365732052657475726e6564207769746820537563636573730300000001000008c0f9974020ef507068183313d0ca808e0d1ca9b2d1ad0c61f5784e7157c362c06536f5ddacdad4451693f48fcc72fff6244073797374656d2d707265706172696e67000300000001000008c0c01f810f36f06cf7f599cf73c5d69c199e61b52892f2c725e847a20593e514e419762ba4546834ffa43564f66154e22e186170702d69645053226fcd7b6bc7ea979b0a550bf49cad993f28e20300000001000008c094c46b0d0e29de7e22a743b54eade2656369002b89a4733a1d3911335431e95eac817c70bf4967662eaf30742a80c64130636f6d706f73652d686173688053226fcd7b6bc7ea979b0a550bf49cad993f28e2370169a9510e56c90b3029dc0300000001000008c0aa25b837d81098d56e726cdfd4664eae9a1269362479753d04bdfd239a45388a056f245841910931ea09ff8278b9d4823c6770752d706f6c6963792d686173688044136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a0300000001000008c057de64e8baecbda8050d4f65922351e9fde5b2d08c7989b987024868d0ff5c8b699bcd919f99425f3f2ada2d2fbcd1da2c696e7374616e63652d696450daf6ebb5452578eba1a13acd1ed103157b03bff60300000001000008c098bd7e6bd3952720b65027fd494834045d06b4a714bf737a06b874638b3ea00ff402f7f583e3e3b05e921c8570433ac630626f6f742d6d722d646f6e65000300000001000008c07f788209b6106d43ad4b0cef85797120c234f175a10b7cb609a73ec1f4285adfe7abca2e1e3d3bd2ba058c2b78d1771d306b65792d70726f76696465725c7b226e616d65223a226e6f6e65222c226964223a22227d0300000001000008c0ba51104636900268b0e059fa3d266419d079d1e94aea26fb9fcbb8d764bf4c89a67ac271b8a0d1a3989945132a111fc72873746f726167652d66730c7a66730300000001000008c01a76b2a80a0be71eae59f80945d876351a7a3fb8e9fd1ff1cede5734aa84ea11fd72b4edfbb6f04e5a85edd114c751bd3073797374656d2d726561647900244073797374656d2d707265706172696e6700186170702d69645053226fcd7b6bc7ea979b0a550bf49cad993f28e230636f6d706f73652d686173688053226fcd7b6bc7ea979b0a550bf49cad993f28e2370169a9510e56c90b3029dc3c6770752d706f6c6963792d686173688044136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a2c696e7374616e63652d696450daf6ebb5452578eba1a13acd1ed103157b03bff630626f6f742d6d722d646f6e6500306b65792d70726f76696465725c7b226e616d65223a226e6f6e65222c226964223a22227d2873746f726167652d66730c7a66733073797374656d2d7265616479004242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424291237b226f735f696d6167655f68617368223a2239386133333934363939353730616466353932316562313762616665643632623337623461363363616432386562353530366234333062363361663265323065222c226370755f636f756e74223a322c226d656d6f72795f73697a65223a323134373438333634382c2271656d755f76657273696f6e223a22382e322e32222c227063695f686f6c6536345f73697a65223a302c22687567657061676573223a66616c73652c226e756d5f67707573223a302c226e756d5f6e767377697463686573223a302c22686f74706c75675f6f6666223a66616c73652c22696d616765223a2264737461636b2d302e362e302d7631617474657374222c22686f73745f73686172655f6d6f6465223a223970222c226f766d665f76617269616e74223a22707265323032353035222c227464785f6174746573746174696f6e5f76617269616e74223a226c697465222c227464785f6d6561737572656d656e74223a7b22636865636b73756d5f66696c65223a224e7a6b774f5759794f446b355957566c4e5749784e54466b5a54426c595445334e5449345a474d314d325a6d596a63794f5467314e574e6c5a4456684e7a5668596d553359574d334d44553059324d315a44466a4e69416762335a745a69356d5a416f795a474d355a6a6c6d4d6a5a684d6d46695a5759784e6a4d794f446b784d54526d4e7a517a4e44686b4e7a45334e6d4a6b4f54466d595467784e54646959575532596a4d324e6a52694d6d56695a544e6d4e6a4a6b49434269656b6c745957646c436a45334d6a49355a6a6c694e6d45795a57526b4d6a55314d4449345a54686c596a63324e4451774e3259324d5467774d44566b4f44426a5932457a4e44426a4f5759344f444a6d5a6d49344e4449334d6d5a6d4d444d6749476c75615852795957316d6379356a63476c764c6d6436436a63774d44686d597a64684d445178596d52694e3255355a6a55785a574a6a4f5751344f5445794d5442684e6a557a4f47566b593249315954517a5a4455325a6d4535593259314d444d344d7a63345a6d5a684e546b674947316c6447466b595852684c6d707a6232344b59544d794f4463784f574a6d595445795a4459304e6a63334e444a69596a5a694e6d466a4e444d785a5751315a4441794f546b34596d5a6a4e47566d5a6a557a4e5759305a4755345a5463774f4745324e6d4d334d53416762575668633356795a57316c626e5175644752344c6d4e696233494b596a63344d6a4d304d7a45324e6a59304f5445324e474a6b5a5755794e444d35595459324e6d4e6d4f4459354f544d775a474d785a4459325a5749314f5441314e444a6d4d6d4d304d544978596a5132596a52694d69416762575668633356795a57316c626e5175633235774c6d4e696233494b5a6a45344e6d497a4e544130596a55324f475269597a59344d546b314f5445344e325a694e6a526c4d6a41324e54566c4d5463334e6a51784e6a51314e7a49785a5463354e5745794d7a42694d54466b4f4749304e53416762575668633356795a57316c626e51755a324e774c6d4e696233494b5a546b334f574a6c5a5449784e546b304e445131596a42684d54417a5a6a46684e7a59334f4468684d6a45324d7a63775a5455775957566d596a526b4d7a41354e47566b4e444d794e5455794d4467354e4745795969416762575668633356795a57316c626e51755958647a4c6d4e696233494b4f444d7a595755334e6a55334e7a4d334e3255344e5442694e324d304e7a526c4d4441305a6a466b5a6a4a6d4e6a6c6a5a47553159324d30596d4d775a544a6d59574e6a4d7a68685a6a41324e5459785a446b785953416762575668633356795a57316c626e51755958647a4c6e4a6c63477868655335716332397543673d3d222c226d6561737572656d656e74223a226f3264325a584a7a61573975424756706257466e5a61526e5932316b62476c755a586b42456d4e76626e4e766247553964485235557a41676157357064443076615735706443427759573570597a30784947356c644335705a6d35686257567a50544167596d6c766332526c646d3568625755394d4342745932553962325a6d4947397663484d396347467561574d6763474e70505735765a57467962486b67636d46755a4739744c6e527964584e3058324e776454313549484a68626d527662533530636e567a6446396962323930624739685a4756795057346764484e6a50584a6c62476c68596d786c494735764c57743262574e7362324e724947527a6447466a61793579623239305a6e4e666147467a614430344e444e6b4d6a5a6b5a4463354d7a51305a574e684f44457a597a55334e32466d597a67354f44526d5a6a466a595463354e4463334d6a4131596d4a694d324d77596a426a4f4451324f574e684f544d335a6a4a6d4947527a6447466a61793579623239305a6e4e6663326c365a5430314f5449794d6a51794e545a7a61325679626d5673583246316447686c626e52705932396b5a56677734724e706930593931364c6d6f4c454f576d6f774b6a73357a686f586c516972527366694d747a70784e3067572f55694b6f6355736e6c73435865724b644e63654268725a584a755a577866614756685a47567958323576636d316862476c365a57543162576c75615852795a46397a6147457a4f4452594d5077305351487239334970317736452b5667693576377475674f574835514a326b4c725279476e4b6f37573369325a73466a786265674c372b574632417a624d4752305a485a6d6f325276646d316d615842795a5449774d6a55774e575274636e526b6f6d747a6157356e624756666347467a633167774731782f6733765475593663704365355668325563567734336e6176594f676e70624d5854426a6d523371584564795534685944694e74636447707a42704953614852336231397759584e7a5744444856366a37486f4d69744c2f6c6e39784c35303854477744694f415970663776366850307462336d6f483750353130473674772b6143457a5833394c4e2b68316d6447526661473969544941514351514142676b434377495145413d3d227d2c22737065635f76657273696f6e223a317d" +} diff --git a/dstack/verifier/fixtures/tdx-lite-v0-get-tls-key.pem b/dstack/verifier/fixtures/tdx-lite-v0-get-tls-key.pem new file mode 100644 index 000000000..9d6c13c62 --- /dev/null +++ b/dstack/verifier/fixtures/tdx-lite-v0-get-tls-key.pem @@ -0,0 +1,376 @@ +-----BEGIN CERTIFICATE----- +MIIjiTCCIzCgAwIBAgIUKC+UzQRG226Kv+X9rJBLEOcfcMcwCgYIKoZIzj0EAwIw +GDEWMBQGA1UEAwwNQXBwIFJvb3QgQ2VydDAeFw03NTAxMDEwMDAwMDBaFw0zNjA5 +MTEwNTM3MjJaMB4xHDAaBgNVBAMME3YwLWF0dGVzdC1lMmUubG9jYWwwWTATBgcq +hkjOPQIBBggqhkjOPQMBBwNCAAS3Z70Z8bB9ztSFCyPuB2eyQOKo1YkI/yE/ist/ +bObDp0kIr2mq0a378GynzArtEY2ga2eXbrunrX1q3oEW5DuIo4IiUDCCIkwwHgYD +VR0RBBcwFYITdjAtYXR0ZXN0LWUyZS5sb2NhbDAPBgNVHQ8BAf8EBQMDB4AAMB0G +A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAkBgorBgEEAYPnPQEDBBYEFFMi +b817a8fql5sKVQv0nK2ZPyjiMBoGCisGAQQBg+c9AQQEDAQKYXBwOmN1c3RvbTCC +IbYGCisGAQQBg+c9AQgEgiGmBIIhogAAOU4EAAIAgQAAAAAAAACTmnIz95xMqZQK +DbOVfwYH3KR5gcDRCV3ngPSRlQvDyQAAAAAOAQQAAAAAAAAAAAAAAAAANGvHehhG +ysIU3S6O3rnuQ0lEnWw/n/LFIUmmNMJ7f9G9MUwu9rlz7uvVV0KVJTGhAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAQAAAAAOcCBgAAAAAAx1eo+x6DIrS/5Z/cS+dPExsA4jgGKX+7+oT9LW95 +qB+z+ddBurcPmghM19/SzfodAVMib817a8fql5sKVQv0nK2ZPyjiNwFpqVEOVskL +MCncAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAA+EONs2uW+F2HUv9/JKiewFx57J7aK6cyyJf7lwyk +KTZbdHGxwFTLhPF7HCsjumZARIzK2T18qJyRh/LQMiNS+OpxVoRaLcIT9EpR3M5S +CQ3i4x7exVVh4kSALHGL6RjkNFLl96lqwQiahIoxWZAuxSrznUhjgKtbGFZeMbIi +2kaZAAgI8q06C8QHwqe68zupGnNKTrTsTPVzw4qk5m/V75yBExYS/G70tI+rIwuG +1zfjZFOhn0CUmZCZmBHl/CBraShmZ6bkiDYGRxg5zt4VgcgbZiaMBdEsbUXKg05Q +bVCjOWg7Ex2zIBGVjRaI25H1TjogrdVQ/gSmVKNRtRCTeswQAABq+Mcpz5zG7Yft +cLkhvo/Tw55jIksWwsxHgN+W+D5W3ZSMtxH1h38zSaMMQ41n0ng9lBBLr0OiyV2P +BFQRCQ4nRF5lXftEmftuPrQ0wydcNA3577iWxW1igRHEuVXBW/MXdMk3J2Nv27yC +NmM7uD8IOIw2hoTTx14prtR+wFDncgYARhAAAAQEGRsE/wAGAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUAAAAAAAAA5wAAAAAAAADlo6e1 +2DDClTuYU0xsWaOjT9w06TP39YmPCoXPCIRrygAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAA3J4qfG+UjxdHTjSn/EPtAw98FWPxur3fY0DILg5UqMUAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAYA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAwgkCLMzE13HcHt4Q2yOsVb40GdIpAaifuMxhOEBDN7IAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKuVbPF4UdIQnWOsv5EeMJDKGmmj +40hcFl8MLi9qNrb4qDD3VdBCahSPAi1qF/dFAOaDdLyIZfP0Atd8Yj6p+/YgAAAB +AgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fBQBeDgAALS0tLS1CRUdJTiBD +RVJUSUZJQ0FURS0tLS0tCk1JSUU4RENDQkplZ0F3SUJBZ0lWQUtXZlI5TXVibE5W +QlZuQlBFSlhhS2RmVnFROU1Bb0dDQ3FHU000OUJBTUMKTUhBeElqQWdCZ05WQkFN +TUdVbHVkR1ZzSUZOSFdDQlFRMHNnVUd4aGRHWnZjbTBnUTBFeEdqQVlCZ05WQkFv +TQpFVWx1ZEdWc0lFTnZjbkJ2Y21GMGFXOXVNUlF3RWdZRFZRUUhEQXRUWVc1MFlT +QkRiR0Z5WVRFTE1Ba0dBMVVFCkNBd0NRMEV4Q3pBSkJnTlZCQVlUQWxWVE1CNFhE +VEkyTURjeU5qQTNOREV3T0ZvWERUTXpNRGN5TmpBM05ERXcKT0Zvd2NERWlNQ0FH +QTFVRUF3d1pTVzUwWld3Z1UwZFlJRkJEU3lCRFpYSjBhV1pwWTJGMFpURWFNQmdH +QTFVRQpDZ3dSU1c1MFpXd2dRMjl5Y0c5eVlYUnBiMjR4RkRBU0JnTlZCQWNNQzFO +aGJuUmhJRU5zWVhKaE1Rc3dDUVlEClZRUUlEQUpEUVRFTE1Ba0dBMVVFQmhNQ1ZW +TXdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUVEKZEttdjgvd0gx +aE5kdnBWOUdNR2NFRCtRS3ZPOCtidmdvdStCY2J3TEMreVgveEJ3eDhNMzlURDRy +YkNmNnZKVwpUeG9EL0ltYXhLKzN4V1VIZHBNSW80SURERENDQXdnd0h3WURWUjBq +QkJnd0ZvQVVsVzlkemIwYjRlbEFTY25VCjlEUE9BVmNMM2xRd2F3WURWUjBmQkdR +d1lqQmdvRjZnWElaYWFIUjBjSE02THk5aGNHa3VkSEoxYzNSbFpITmwKY25acFky +VnpMbWx1ZEdWc0xtTnZiUzl6WjNndlkyVnlkR2xtYVdOaGRHbHZiaTkyTkM5d1ky +dGpjbXcvWTJFOQpjR3hoZEdadmNtMG1aVzVqYjJScGJtYzlaR1Z5TUIwR0ExVWRE +Z1FXQkJRWkNTbFRuazcvRGt0bXAvUEptME9QCnpBdFU1ekFPQmdOVkhROEJBZjhF +QkFNQ0JzQXdEQVlEVlIwVEFRSC9CQUl3QURDQ0Fqa0dDU3FHU0liNFRRRU4KQVFT +Q0Fpb3dnZ0ltTUI0R0NpcUdTSWI0VFFFTkFRRUVFRzlnODV5R2RRQ3dWRFhzZ0Z5 +Z0crb3dnZ0ZqQmdvcQpoa2lHK0UwQkRRRUNNSUlCVXpBUUJnc3Foa2lHK0UwQkRR +RUNBUUlCQkRBUUJnc3Foa2lHK0UwQkRRRUNBZ0lCCkJEQVFCZ3NxaGtpRytFMEJE +UUVDQXdJQkFqQVFCZ3NxaGtpRytFMEJEUUVDQkFJQkFqQVFCZ3NxaGtpRytFMEIK +RFFFQ0JRSUJCREFRQmdzcWhraUcrRTBCRFFFQ0JnSUJBVEFRQmdzcWhraUcrRTBC +RFFFQ0J3SUJBREFRQmdzcQpoa2lHK0UwQkRRRUNDQUlCQlRBUUJnc3Foa2lHK0Uw +QkRRRUNDUUlCQURBUUJnc3Foa2lHK0UwQkRRRUNDZ0lCCkFEQVFCZ3NxaGtpRytF +MEJEUUVDQ3dJQkFEQVFCZ3NxaGtpRytFMEJEUUVDREFJQkFEQVFCZ3NxaGtpRytF +MEIKRFFFQ0RRSUJBREFRQmdzcWhraUcrRTBCRFFFQ0RnSUJBREFRQmdzcWhraUcr +RTBCRFFFQ0R3SUJBREFRQmdzcQpoa2lHK0UwQkRRRUNFQUlCQURBUUJnc3Foa2lH +K0UwQkRRRUNFUUlCRFRBZkJnc3Foa2lHK0UwQkRRRUNFZ1FRCkJBUUNBZ1FCQUFV +QUFBQUFBQUFBQURBUUJnb3Foa2lHK0UwQkRRRURCQUlBQURBVUJnb3Foa2lHK0Uw +QkRRRUUKQkFhUXdHOEFBQUF3RHdZS0tvWklodmhOQVEwQkJRb0JBVEFlQmdvcWhr +aUcrRTBCRFFFR0JCQjhyRzF1TFpQaQo4OVhFaWk1VUFBVGhNRVFHQ2lxR1NJYjRU +UUVOQVFjd05qQVFCZ3NxaGtpRytFMEJEUUVIQVFFQi96QVFCZ3NxCmhraUcrRTBC +RFFFSEFnRUIvekFRQmdzcWhraUcrRTBCRFFFSEF3RUIvekFLQmdncWhrak9QUVFE +QWdOSEFEQkUKQWlBSm1rdzdnRUgrY20vREZmcnVmSzg0N090dVRkZTIvdTVhUDFz +L0JhcjdQQUlnU2Q4TFAxUWF0OEdQamo0ZQpoTCsxVHBxTmRVei9ZRnMwU1FSeVRZ +QUZndDQ9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0tLS1CRUdJTiBDRVJU +SUZJQ0FURS0tLS0tCk1JSUNsakNDQWoyZ0F3SUJBZ0lWQUpWdlhjMjlHK0hwUUVu +SjFQUXp6Z0ZYQzk1VU1Bb0dDQ3FHU000OUJBTUMKTUdneEdqQVlCZ05WQkFNTUVV +bHVkR1ZzSUZOSFdDQlNiMjkwSUVOQk1Sb3dHQVlEVlFRS0RCRkpiblJsYkNCRApi +M0p3YjNKaGRHbHZiakVVTUJJR0ExVUVCd3dMVTJGdWRHRWdRMnhoY21FeEN6QUpC +Z05WQkFnTUFrTkJNUXN3CkNRWURWUVFHRXdKVlV6QWVGdzB4T0RBMU1qRXhNRFV3 +TVRCYUZ3MHpNekExTWpFeE1EVXdNVEJhTUhBeElqQWcKQmdOVkJBTU1HVWx1ZEdW +c0lGTkhXQ0JRUTBzZ1VHeGhkR1p2Y20wZ1EwRXhHakFZQmdOVkJBb01FVWx1ZEdW +cwpJRU52Y25CdmNtRjBhVzl1TVJRd0VnWURWUVFIREF0VFlXNTBZU0JEYkdGeVlU +RUxNQWtHQTFVRUNBd0NRMEV4CkN6QUpCZ05WQkFZVEFsVlRNRmt3RXdZSEtvWkl6 +ajBDQVFZSUtvWkl6ajBEQVFjRFFnQUVOU0IvN3QyMWxYU08KMkN1enB4dzc0ZUpC +NzJFeURHZ1c1clhDdHgydFZUTHE2aEtrNnorVWlSWkNucVI3cHNPdmdxRmVTeGxt +VGxKbAplVG1pMldZejNxT0J1ekNCdURBZkJnTlZIU01FR0RBV2dCUWlaUXpXV3Aw +MGlmT0R0SlZTdjFBYk9TY0dyREJTCkJnTlZIUjhFU3pCSk1FZWdSYUJEaGtGb2RI +Undjem92TDJObGNuUnBabWxqWVhSbGN5NTBjblZ6ZEdWa2MyVnkKZG1salpYTXVh +VzUwWld3dVkyOXRMMGx1ZEdWc1UwZFlVbTl2ZEVOQkxtUmxjakFkQmdOVkhRNEVG +Z1FVbFc5ZAp6YjBiNGVsQVNjblU5RFBPQVZjTDNsUXdEZ1lEVlIwUEFRSC9CQVFE +QWdFR01CSUdBMVVkRXdFQi93UUlNQVlCCkFmOENBUUF3Q2dZSUtvWkl6ajBFQXdJ +RFJ3QXdSQUlnWHNWa2kwdytpNlZZR1czVUYvMjJ1YVhlMFlKRGoxVWUKbkErVGpE +MWFpNWNDSUNZYjFTQW1ENXhrZlRWcHZvNFVveWlTWXhyRFdMbVVSNENJOU5LeWZQ +TisKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQotLS0tLUJFR0lOIENFUlRJRklD +QVRFLS0tLS0KTUlJQ2p6Q0NBalNnQXdJQkFnSVVJbVVNMWxxZE5JbnpnN1NWVXI5 +UUd6a25CcXd3Q2dZSUtvWkl6ajBFQXdJdwphREVhTUJnR0ExVUVBd3dSU1c1MFpX +d2dVMGRZSUZKdmIzUWdRMEV4R2pBWUJnTlZCQW9NRVVsdWRHVnNJRU52CmNuQnZj +bUYwYVc5dU1SUXdFZ1lEVlFRSERBdFRZVzUwWVNCRGJHRnlZVEVMTUFrR0ExVUVD +QXdDUTBFeEN6QUoKQmdOVkJBWVRBbFZUTUI0WERURTRNRFV5TVRFd05EVXhNRm9Y +RFRRNU1USXpNVEl6TlRrMU9Wb3dhREVhTUJnRwpBMVVFQXd3UlNXNTBaV3dnVTBk +WUlGSnZiM1FnUTBFeEdqQVlCZ05WQkFvTUVVbHVkR1ZzSUVOdmNuQnZjbUYwCmFX +OXVNUlF3RWdZRFZRUUhEQXRUWVc1MFlTQkRiR0Z5WVRFTE1Ba0dBMVVFQ0F3Q1Ew +RXhDekFKQmdOVkJBWVQKQWxWVE1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERB +UWNEUWdBRUM2bkV3TURJWVpPai9pUFdzQ3phRUtpNwoxT2lPU0xSRmhXR2pibkJW +SmZWbmtZNHUzSWprRFlZTDBNeE80bXFzeVlqbEJhbFRWWXhGUDJzSkJLNXpsS09C +CnV6Q0J1REFmQmdOVkhTTUVHREFXZ0JRaVpReldXcDAwaWZPRHRKVlN2MUFiT1Nj +R3JEQlNCZ05WSFI4RVN6QkoKTUVlZ1JhQkRoa0ZvZEhSd2N6b3ZMMk5sY25ScFpt +bGpZWFJsY3k1MGNuVnpkR1ZrYzJWeWRtbGpaWE11YVc1MApaV3d1WTI5dEwwbHVk +R1ZzVTBkWVVtOXZkRU5CTG1SbGNqQWRCZ05WSFE0RUZnUVVJbVVNMWxxZE5Jbnpn +N1NWClVyOVFHemtuQnF3d0RnWURWUjBQQVFIL0JBUURBZ0VHTUJJR0ExVWRFd0VC +L3dRSU1BWUJBZjhDQVFFd0NnWUkKS29aSXpqMEVBd0lEU1FBd1JnSWhBT1cvNVFr +UitTOUNpU0RjTm9vd0x1UFJMc1dHZi9ZaTdHU1g5NEJnd1R3ZwpBaUVBNEowbHJI +b01zK1hvNW8vc1g2TzlRV3hIUkF2WlVHT2RSUTdjdnFSWGFxST0KLS0tLS1FTkQg +Q0VSVElGSUNBVEUtLS0tLQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAACgAA +AMAJXQTPJv4DrvbjVh+iTBqhzqk/Sur1Y7H592FhhMU0VIdZJXWUNHac7CSQrLVj +ozcsYWNwaS1sb2FkZXIkQUNQSSBEQVRBAAAAAAoAAADAjZpNR3ehvHfs2djTekYo +EpqABSpRAyAVmiCpI70HoOkNjR8uHr8IiZKyXw0PpnLvJGFjcGktcnNkcCRBQ1BJ +IERBVEEAAAAACgAAAMAwcHIeFpvEGIRyTLDmswguG68kkIPYs4kYG6ULmvqVEFeH +bDgLiHDowvrPLv9norYsYWNwaS10YWJsZXMkQUNQSSBEQVRBAwAAAAEAAAjA+ZdA +IO9QcGgYMxPQyoCODRypstGtDGH1eE5xV8NiwGU29d2s2tRFFpP0j8xy//YkQHN5 +c3RlbS1wcmVwYXJpbmcAAwAAAAEAAAjAwB+BDzbwbPf1mc9zxdacGZ5htSiS8scl +6EeiBZPlFOQZdiukVGg0/6Q1ZPZhVOIuGGFwcC1pZFBTIm/Ne2vH6pebClUL9Jyt +mT8o4gMAAAABAAAIwJTEaw0OKd5+IqdDtU6t4mVjaQAriaRzOh05ETNUMelerIF8 +cL9JZ2YurzB0KoDGQTBjb21wb3NlLWhhc2iAUyJvzXtrx+qXmwpVC/ScrZk/KOI3 +AWmpUQ5WyQswKdwDAAAAAQAACMCqJbg32BCY1W5ybN/UZk6umhJpNiR5dT0Evf0j +mkU4igVvJFhBkQkx6gn/gni51II8Z3B1LXBvbGljeS1oYXNogEQTb6NVs2eKEUat +FvfoZJ6U+0/CH+d+gxDAYPYcqv+KAwAAAAEAAAjAV95k6LrsvagFDU9lkiNR6f3l +stCMeYm5hwJIaND/XItpm82Rn5lCXz8q2i0vvNHaLGluc3RhbmNlLWlkUNr267VF +JXjroaE6zR7RAxV7A7/2AwAAAAEAAAjAmL1+a9OVJyC2UCf9SUg0BF0GtKcUv3N6 +Brh0Y4s+oA/0Avf1g+PjsF6SHIVwQzrGMGJvb3QtbXItZG9uZQADAAAAAQAACMB/ +eIIJthBtQ61LDO+FeXEgwjTxdaELfLYJpz7B9Cha3+eryi4ePTvSugWMK3jRdx0w +a2V5LXByb3ZpZGVyXHsibmFtZSI6Im5vbmUiLCJpZCI6IiJ9AwAAAAEAAAjAulEQ +RjaQAmiw4Fn6PSZkGdB50elK6ib7n8u412S/TImmesJxuKDRo5iZRRMqER/HKHN0 +b3JhZ2UtZnMMemZzAwAAAAEAAAjAGnayqAoL5x6uWfgJRdh2NRp6P7jp/R/xzt5X +NKqE6hH9crTt+7bwTlqF7dEUx1G9MHN5c3RlbS1yZWFkeQAkQHN5c3RlbS1wcmVw +YXJpbmcAGGFwcC1pZFBTIm/Ne2vH6pebClUL9JytmT8o4jBjb21wb3NlLWhhc2iA +UyJvzXtrx+qXmwpVC/ScrZk/KOI3AWmpUQ5WyQswKdw8Z3B1LXBvbGljeS1oYXNo +gEQTb6NVs2eKEUatFvfoZJ6U+0/CH+d+gxDAYPYcqv+KLGluc3RhbmNlLWlkUNr2 +67VFJXjroaE6zR7RAxV7A7/2MGJvb3QtbXItZG9uZQAwa2V5LXByb3ZpZGVyXHsi +bmFtZSI6Im5vbmUiLCJpZCI6IiJ9KHN0b3JhZ2UtZnMMemZzMHN5c3RlbS1yZWFk +eQBpKGZnpuSINgZHGDnO3hWByBtmJowF0SxtRcqDTlBtUKM5aDsTHbMgEZWNFojb +kfVOOiCt1VD+BKZUo1G1EJN6kSN7Im9zX2ltYWdlX2hhc2giOiI5OGEzMzk0Njk5 +NTcwYWRmNTkyMWViMTdiYWZlZDYyYjM3YjRhNjNjYWQyOGViNTUwNmI0MzBiNjNh +ZjJlMjBlIiwiY3B1X2NvdW50IjoyLCJtZW1vcnlfc2l6ZSI6MjE0NzQ4MzY0OCwi +cWVtdV92ZXJzaW9uIjoiOC4yLjIiLCJwY2lfaG9sZTY0X3NpemUiOjAsImh1Z2Vw +YWdlcyI6ZmFsc2UsIm51bV9ncHVzIjowLCJudW1fbnZzd2l0Y2hlcyI6MCwiaG90 +cGx1Z19vZmYiOmZhbHNlLCJpbWFnZSI6ImRzdGFjay0wLjYuMC12MWF0dGVzdCIs +Imhvc3Rfc2hhcmVfbW9kZSI6IjlwIiwib3ZtZl92YXJpYW50IjoicHJlMjAyNTA1 +IiwidGR4X2F0dGVzdGF0aW9uX3ZhcmlhbnQiOiJsaXRlIiwidGR4X21lYXN1cmVt +ZW50Ijp7ImNoZWNrc3VtX2ZpbGUiOiJOemt3T1dZeU9EazVZV1ZsTldJeE5URmta +VEJsWVRFM05USTRaR00xTTJabVlqY3lPVGcxTldObFpEVmhOelZoWW1VM1lXTTNN +RFUwWTJNMVpERmpOaUFnYjNadFppNW1aQW95WkdNNVpqbG1NalpoTW1GaVpXWXhO +ak15T0RreE1UUm1OelF6TkRoa056RTNObUprT1RGbVlUZ3hOVGRpWVdVMllqTTJO +alJpTW1WaVpUTm1OakprSUNCaWVrbHRZV2RsQ2pFM01qSTVaamxpTm1FeVpXUmtN +alUxTURJNFpUaGxZamMyTkRRd04yWTJNVGd3TURWa09EQmpZMkV6TkRCak9XWTRP +REptWm1JNE5ESTNNbVptTURNZ0lHbHVhWFJ5WVcxbWN5NWpjR2x2TG1kNkNqY3dN +RGhtWXpkaE1EUXhZbVJpTjJVNVpqVXhaV0pqT1dRNE9URXlNVEJoTmpVek9HVmtZ +MkkxWVRRelpEVTJabUU1WTJZMU1ETTRNemM0Wm1aaE5Ua2dJRzFsZEdGa1lYUmhM +bXB6YjI0S1lUTXlPRGN4T1dKbVlURXlaRFkwTmpjM05ESmlZalppTm1Gak5ETXha +V1ExWkRBeU9UazRZbVpqTkdWbVpqVXpOV1kwWkdVNFpUY3dPR0UyTm1NM01TQWdi +V1ZoYzNWeVpXMWxiblF1ZEdSNExtTmliM0lLWWpjNE1qTTBNekUyTmpZME9URTJO +R0prWldVeU5ETTVZVFkyTm1ObU9EWTVPVE13WkdNeFpEWTJaV0kxT1RBMU5ESm1N +bU0wTVRJeFlqUTJZalJpTWlBZ2JXVmhjM1Z5WlcxbGJuUXVjMjV3TG1OaWIzSUta +akU0Tm1Jek5UQTBZalUyT0dSaVl6WTRNVGsxT1RFNE4yWmlOalJsTWpBMk5UVmxN +VGMzTmpReE5qUTFOekl4WlRjNU5XRXlNekJpTVRGa09HSTBOU0FnYldWaGMzVnla +VzFsYm5RdVoyTndMbU5pYjNJS1pUazNPV0psWlRJeE5UazBORFExWWpCaE1UQXpa +akZoTnpZM09EaGhNakUyTXpjd1pUVXdZV1ZtWWpSa016QTVOR1ZrTkRNeU5UVXlN +RGc1TkdFeVlpQWdiV1ZoYzNWeVpXMWxiblF1WVhkekxtTmliM0lLT0RNellXVTNO +alUzTnpNM04yVTROVEJpTjJNME56UmxNREEwWmpGa1pqSm1OamxqWkdVMVkyTTBZ +bU13WlRKbVlXTmpNemhoWmpBMk5UWXhaRGt4WVNBZ2JXVmhjM1Z5WlcxbGJuUXVZ +WGR6TG5KbGNHeGhlUzVxYzI5dUNnPT0iLCJtZWFzdXJlbWVudCI6Im8yZDJaWEp6 +YVc5dUJHVnBiV0ZuWmFSblkyMWtiR2x1WlhrQkVtTnZibk52YkdVOWRIUjVVekFn +YVc1cGREMHZhVzVwZENCd1lXNXBZejB4SUc1bGRDNXBabTVoYldWelBUQWdZbWx2 +YzJSbGRtNWhiV1U5TUNCdFkyVTliMlptSUc5dmNITTljR0Z1YVdNZ2NHTnBQVzV2 +WldGeWJIa2djbUZ1Wkc5dExuUnlkWE4wWDJOd2RUMTVJSEpoYm1SdmJTNTBjblZ6 +ZEY5aWIyOTBiRzloWkdWeVBXNGdkSE5qUFhKbGJHbGhZbXhsSUc1dkxXdDJiV05z +YjJOcklHUnpkR0ZqYXk1eWIyOTBabk5mYUdGemFEMDRORE5rTWpaa1pEYzVNelEw +WldOaE9ERXpZelUzTjJGbVl6ZzVPRFJtWmpGallUYzVORGMzTWpBMVltSmlNMk13 +WWpCak9EUTJPV05oT1RNM1pqSm1JR1J6ZEdGamF5NXliMjkwWm5OZmMybDZaVDAx +T1RJeU1qUXlOVFp6YTJWeWJtVnNYMkYxZEdobGJuUnBZMjlrWlZndzRyTnBpMFk5 +MTZMbW9MRU9XbW93S2pzNXpob1hsUWlyUnNmaU10enB4TjBnVy9VaUtvY1Vzbmxz +Q1hlcktkTmNlQmhyWlhKdVpXeGZhR1ZoWkdWeVgyNXZjbTFoYkdsNlpXVDFiV2x1 +YVhSeVpGOXphR0V6T0RSWU1QdzBTUUhyOTNJcDF3NkUrVmdpNXY3dHVnT1dINVFK +MmtMclJ5R25LbzdXM2kyWnNGanhiZWdMNytXRjJBemJNR1IwWkhabW8yUnZkbTFt +YVhCeVpUSXdNalV3TldSdGNuUmtvbXR6YVc1bmJHVmZjR0Z6YzFnd0cxeC9nM3ZU +dVk2Y3BDZTVWaDJVY1Z3NDNuYXZZT2ducGJNWFRCam1SM3FYRWR5VTRoWURpTnRj +ZEdwekJwSVNhSFIzYjE5d1lYTnpXRERIVjZqN0hvTWl0TC9sbjl4TDUwOFRHd0Rp +T0FZcGY3djZoUDB0YjNtb0g3UDUxMEc2dHcrYUNFelgzOUxOK2gxbWRHUmZhRzlp +VElBUUNRUUFCZ2tDQ3dJUUVBPT0ifSwic3BlY192ZXJzaW9uIjoxfTAKBggqhkjO +PQQDAgNHADBEAiAnZ1F51gI1Ns999vSmppuRqOtjCnuWdmN/y962nKXrJgIgaGVV ++VLnBegRapATXSNeg4lwwHCjEOXAQOyMj36ulfU= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIiCDCCIa2gAwIBAgIUHLqInDbCs4Sejnc06Isj8lqf50wwCgYIKoZIzj0EAwIw +GDEWMBQGA1UEAwwNQXBwIFJvb3QgQ2VydDAeFw03NTAxMDEwMDAwMDBaFw0zNjA5 +MTEwNTM3MDFaMBgxFjAUBgNVBAMMDUFwcCBSb290IENlcnQwWTATBgcqhkjOPQIB +BggqhkjOPQMBBwNCAAQX2ft1fUcsVG1ySNCRw8384dCUkTO0pqhsk63HhtNEVuJo +FtanJnITtkJ3Hs70XCs4xOeMXiHmY2GThIt+o9dJo4Ig0zCCIM8wDwYDVR0PAQH/ +BAUDAweGADAdBgNVHQ4EFgQUwnjZm59enGH8qUSDLplIxRcLs3QwEgYDVR0TAQH/ +BAgwBgEB/wIBATCCIIcGCisGAQQBg+c9AQgEgiB3BIIgcwAAOU4EAAIAgQAAAAAA +AACTmnIz95xMqZQKDbOVfwYH3KR5gcDRCV3ngPSRlQvDyQAAAAAOAQQAAAAAAAAA +AAAAAAAANGvHehhGysIU3S6O3rnuQ0lEnWw/n/LFIUmmNMJ7f9G9MUwu9rlz7uvV +V0KVJTGhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAQAAAAAOcCBgAAAAAAx1eo+x6DIrS/5Z/cS+dPExsA +4jgGKX+7+oT9LW95qB+z+ddBurcPmghM19/SzfodAVMib817a8fql5sKVQv0nK2Z +PyjiNwFpqVEOVskLMCncAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+EONs2uW+F2HUv9/JKiewFx5 +7J7aK6cyyJf7lwykKTZbdHGxwFTLhPF7HCsjumZARIzK2T18qJyRh/LQMiNS+Opx +VoRaLcIT9EpR3M5SCQ3i4x7exVVh4kSALHGL6RjkNFLl96lqwQiahIoxWZAuxSrz +nUhjgKtbGFZeMbIi2kaZAAgI8q06C8QHwqe68zupZkNAx3O/3NL5/2I79JgBl4bX +DCrywo1BSZTG/9iNGISpfuXu66tOpjgtioUOHIKy/rjB/9oPaTk340RIB59CnmbW +IqjHhYD9M7bPvyJVMfY6nhmDWBkLoVcyeEfEkF3VUNzA0+OWLNwzZkxGKLx0UMwQ +AACY+yzK1tbOk6XtbgicG8Ll6H0NtPMmddtHMQPkC5tY4XUViZmSzK2bVINZ6NJ4 +YZqYDo1nCw4fkZKhEwZVGBohRF5lXftEmftuPrQ0wydcNA3577iWxW1igRHEuVXB +W/MXdMk3J2Nv27yCNmM7uD8IOIw2hoTTx14prtR+wFDncgYARhAAAAQEGRsE/wAG +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUAAAAAAAAA +5wAAAAAAAADlo6e12DDClTuYU0xsWaOjT9w06TP39YmPCoXPCIRrygAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA3J4qfG+UjxdHTjSn/EPtAw98FWPxur3f +Y0DILg5UqMUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAACAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwgkCLMzE13HcHt4Q2yOsVb40GdIpAaif +uMxhOEBDN7IAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPpF3NAxz1HI +T1c68zZak2VTQLTHTnbcL0mMAO769fAEmjy0agPiZMwWp/9yLwdf2sV8WPX9PzNo +yFm4qfKxcdQgAAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fBQBeDgAA +LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUU4RENDQkplZ0F3SUJBZ0lW +QUtXZlI5TXVibE5WQlZuQlBFSlhhS2RmVnFROU1Bb0dDQ3FHU000OUJBTUMKTUhB +eElqQWdCZ05WQkFNTUdVbHVkR1ZzSUZOSFdDQlFRMHNnVUd4aGRHWnZjbTBnUTBF +eEdqQVlCZ05WQkFvTQpFVWx1ZEdWc0lFTnZjbkJ2Y21GMGFXOXVNUlF3RWdZRFZR +UUhEQXRUWVc1MFlTQkRiR0Z5WVRFTE1Ba0dBMVVFCkNBd0NRMEV4Q3pBSkJnTlZC +QVlUQWxWVE1CNFhEVEkyTURjeU5qQTNOREV3T0ZvWERUTXpNRGN5TmpBM05ERXcK +T0Zvd2NERWlNQ0FHQTFVRUF3d1pTVzUwWld3Z1UwZFlJRkJEU3lCRFpYSjBhV1pw +WTJGMFpURWFNQmdHQTFVRQpDZ3dSU1c1MFpXd2dRMjl5Y0c5eVlYUnBiMjR4RkRB +U0JnTlZCQWNNQzFOaGJuUmhJRU5zWVhKaE1Rc3dDUVlEClZRUUlEQUpEUVRFTE1B +a0dBMVVFQmhNQ1ZWTXdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FB +UVEKZEttdjgvd0gxaE5kdnBWOUdNR2NFRCtRS3ZPOCtidmdvdStCY2J3TEMreVgv +eEJ3eDhNMzlURDRyYkNmNnZKVwpUeG9EL0ltYXhLKzN4V1VIZHBNSW80SUREREND +QXdnd0h3WURWUjBqQkJnd0ZvQVVsVzlkemIwYjRlbEFTY25VCjlEUE9BVmNMM2xR +d2F3WURWUjBmQkdRd1lqQmdvRjZnWElaYWFIUjBjSE02THk5aGNHa3VkSEoxYzNS +bFpITmwKY25acFkyVnpMbWx1ZEdWc0xtTnZiUzl6WjNndlkyVnlkR2xtYVdOaGRH +bHZiaTkyTkM5d1kydGpjbXcvWTJFOQpjR3hoZEdadmNtMG1aVzVqYjJScGJtYzla +R1Z5TUIwR0ExVWREZ1FXQkJRWkNTbFRuazcvRGt0bXAvUEptME9QCnpBdFU1ekFP +QmdOVkhROEJBZjhFQkFNQ0JzQXdEQVlEVlIwVEFRSC9CQUl3QURDQ0Fqa0dDU3FH +U0liNFRRRU4KQVFTQ0Fpb3dnZ0ltTUI0R0NpcUdTSWI0VFFFTkFRRUVFRzlnODV5 +R2RRQ3dWRFhzZ0Z5Z0crb3dnZ0ZqQmdvcQpoa2lHK0UwQkRRRUNNSUlCVXpBUUJn +c3Foa2lHK0UwQkRRRUNBUUlCQkRBUUJnc3Foa2lHK0UwQkRRRUNBZ0lCCkJEQVFC +Z3NxaGtpRytFMEJEUUVDQXdJQkFqQVFCZ3NxaGtpRytFMEJEUUVDQkFJQkFqQVFC +Z3NxaGtpRytFMEIKRFFFQ0JRSUJCREFRQmdzcWhraUcrRTBCRFFFQ0JnSUJBVEFR +QmdzcWhraUcrRTBCRFFFQ0J3SUJBREFRQmdzcQpoa2lHK0UwQkRRRUNDQUlCQlRB +UUJnc3Foa2lHK0UwQkRRRUNDUUlCQURBUUJnc3Foa2lHK0UwQkRRRUNDZ0lCCkFE +QVFCZ3NxaGtpRytFMEJEUUVDQ3dJQkFEQVFCZ3NxaGtpRytFMEJEUUVDREFJQkFE +QVFCZ3NxaGtpRytFMEIKRFFFQ0RRSUJBREFRQmdzcWhraUcrRTBCRFFFQ0RnSUJB +REFRQmdzcWhraUcrRTBCRFFFQ0R3SUJBREFRQmdzcQpoa2lHK0UwQkRRRUNFQUlC +QURBUUJnc3Foa2lHK0UwQkRRRUNFUUlCRFRBZkJnc3Foa2lHK0UwQkRRRUNFZ1FR +CkJBUUNBZ1FCQUFVQUFBQUFBQUFBQURBUUJnb3Foa2lHK0UwQkRRRURCQUlBQURB +VUJnb3Foa2lHK0UwQkRRRUUKQkFhUXdHOEFBQUF3RHdZS0tvWklodmhOQVEwQkJR +b0JBVEFlQmdvcWhraUcrRTBCRFFFR0JCQjhyRzF1TFpQaQo4OVhFaWk1VUFBVGhN +RVFHQ2lxR1NJYjRUUUVOQVFjd05qQVFCZ3NxaGtpRytFMEJEUUVIQVFFQi96QVFC +Z3NxCmhraUcrRTBCRFFFSEFnRUIvekFRQmdzcWhraUcrRTBCRFFFSEF3RUIvekFL +QmdncWhrak9QUVFEQWdOSEFEQkUKQWlBSm1rdzdnRUgrY20vREZmcnVmSzg0N090 +dVRkZTIvdTVhUDFzL0JhcjdQQUlnU2Q4TFAxUWF0OEdQamo0ZQpoTCsxVHBxTmRV +ei9ZRnMwU1FSeVRZQUZndDQ9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0t +LS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNsakNDQWoyZ0F3SUJBZ0lWQUpW +dlhjMjlHK0hwUUVuSjFQUXp6Z0ZYQzk1VU1Bb0dDQ3FHU000OUJBTUMKTUdneEdq +QVlCZ05WQkFNTUVVbHVkR1ZzSUZOSFdDQlNiMjkwSUVOQk1Sb3dHQVlEVlFRS0RC +RkpiblJsYkNCRApiM0p3YjNKaGRHbHZiakVVTUJJR0ExVUVCd3dMVTJGdWRHRWdR +MnhoY21FeEN6QUpCZ05WQkFnTUFrTkJNUXN3CkNRWURWUVFHRXdKVlV6QWVGdzB4 +T0RBMU1qRXhNRFV3TVRCYUZ3MHpNekExTWpFeE1EVXdNVEJhTUhBeElqQWcKQmdO +VkJBTU1HVWx1ZEdWc0lGTkhXQ0JRUTBzZ1VHeGhkR1p2Y20wZ1EwRXhHakFZQmdO +VkJBb01FVWx1ZEdWcwpJRU52Y25CdmNtRjBhVzl1TVJRd0VnWURWUVFIREF0VFlX +NTBZU0JEYkdGeVlURUxNQWtHQTFVRUNBd0NRMEV4CkN6QUpCZ05WQkFZVEFsVlRN +Rmt3RXdZSEtvWkl6ajBDQVFZSUtvWkl6ajBEQVFjRFFnQUVOU0IvN3QyMWxYU08K +MkN1enB4dzc0ZUpCNzJFeURHZ1c1clhDdHgydFZUTHE2aEtrNnorVWlSWkNucVI3 +cHNPdmdxRmVTeGxtVGxKbAplVG1pMldZejNxT0J1ekNCdURBZkJnTlZIU01FR0RB +V2dCUWlaUXpXV3AwMGlmT0R0SlZTdjFBYk9TY0dyREJTCkJnTlZIUjhFU3pCSk1F +ZWdSYUJEaGtGb2RIUndjem92TDJObGNuUnBabWxqWVhSbGN5NTBjblZ6ZEdWa2My +VnkKZG1salpYTXVhVzUwWld3dVkyOXRMMGx1ZEdWc1UwZFlVbTl2ZEVOQkxtUmxj +akFkQmdOVkhRNEVGZ1FVbFc5ZAp6YjBiNGVsQVNjblU5RFBPQVZjTDNsUXdEZ1lE +VlIwUEFRSC9CQVFEQWdFR01CSUdBMVVkRXdFQi93UUlNQVlCCkFmOENBUUF3Q2dZ +SUtvWkl6ajBFQXdJRFJ3QXdSQUlnWHNWa2kwdytpNlZZR1czVUYvMjJ1YVhlMFlK +RGoxVWUKbkErVGpEMWFpNWNDSUNZYjFTQW1ENXhrZlRWcHZvNFVveWlTWXhyRFdM +bVVSNENJOU5LeWZQTisKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQotLS0tLUJF +R0lOIENFUlRJRklDQVRFLS0tLS0KTUlJQ2p6Q0NBalNnQXdJQkFnSVVJbVVNMWxx +ZE5JbnpnN1NWVXI5UUd6a25CcXd3Q2dZSUtvWkl6ajBFQXdJdwphREVhTUJnR0Ex +VUVBd3dSU1c1MFpXd2dVMGRZSUZKdmIzUWdRMEV4R2pBWUJnTlZCQW9NRVVsdWRH +VnNJRU52CmNuQnZjbUYwYVc5dU1SUXdFZ1lEVlFRSERBdFRZVzUwWVNCRGJHRnlZ +VEVMTUFrR0ExVUVDQXdDUTBFeEN6QUoKQmdOVkJBWVRBbFZUTUI0WERURTRNRFV5 +TVRFd05EVXhNRm9YRFRRNU1USXpNVEl6TlRrMU9Wb3dhREVhTUJnRwpBMVVFQXd3 +UlNXNTBaV3dnVTBkWUlGSnZiM1FnUTBFeEdqQVlCZ05WQkFvTUVVbHVkR1ZzSUVO +dmNuQnZjbUYwCmFXOXVNUlF3RWdZRFZRUUhEQXRUWVc1MFlTQkRiR0Z5WVRFTE1B +a0dBMVVFQ0F3Q1EwRXhDekFKQmdOVkJBWVQKQWxWVE1Ga3dFd1lIS29aSXpqMENB +UVlJS29aSXpqMERBUWNEUWdBRUM2bkV3TURJWVpPai9pUFdzQ3phRUtpNwoxT2lP +U0xSRmhXR2pibkJWSmZWbmtZNHUzSWprRFlZTDBNeE80bXFzeVlqbEJhbFRWWXhG +UDJzSkJLNXpsS09CCnV6Q0J1REFmQmdOVkhTTUVHREFXZ0JRaVpReldXcDAwaWZP +RHRKVlN2MUFiT1NjR3JEQlNCZ05WSFI4RVN6QkoKTUVlZ1JhQkRoa0ZvZEhSd2N6 +b3ZMMk5sY25ScFptbGpZWFJsY3k1MGNuVnpkR1ZrYzJWeWRtbGpaWE11YVc1MApa +V3d1WTI5dEwwbHVkR1ZzVTBkWVVtOXZkRU5CTG1SbGNqQWRCZ05WSFE0RUZnUVVJ +bVVNMWxxZE5JbnpnN1NWClVyOVFHemtuQnF3d0RnWURWUjBQQVFIL0JBUURBZ0VH +TUJJR0ExVWRFd0VCL3dRSU1BWUJBZjhDQVFFd0NnWUkKS29aSXpqMEVBd0lEU1FB +d1JnSWhBT1cvNVFrUitTOUNpU0RjTm9vd0x1UFJMc1dHZi9ZaTdHU1g5NEJnd1R3 +ZwpBaUVBNEowbHJIb01zK1hvNW8vc1g2TzlRV3hIUkF2WlVHT2RSUTdjdnFSWGFx +ST0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQoAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAACQAAAAACgAAAMAJXQTPJv4DrvbjVh+iTBqhzqk/Sur1Y7H592FhhMU0VIdZ +JXWUNHac7CSQrLVjozcsYWNwaS1sb2FkZXIkQUNQSSBEQVRBAAAAAAoAAADAjZpN +R3ehvHfs2djTekYoEpqABSpRAyAVmiCpI70HoOkNjR8uHr8IiZKyXw0PpnLvJGFj +cGktcnNkcCRBQ1BJIERBVEEAAAAACgAAAMAwcHIeFpvEGIRyTLDmswguG68kkIPY +s4kYG6ULmvqVEFeHbDgLiHDowvrPLv9norYsYWNwaS10YWJsZXMkQUNQSSBEQVRB +AwAAAAEAAAjA+ZdAIO9QcGgYMxPQyoCODRypstGtDGH1eE5xV8NiwGU29d2s2tRF +FpP0j8xy//YkQHN5c3RlbS1wcmVwYXJpbmcAAwAAAAEAAAjAwB+BDzbwbPf1mc9z +xdacGZ5htSiS8scl6EeiBZPlFOQZdiukVGg0/6Q1ZPZhVOIuGGFwcC1pZFBTIm/N +e2vH6pebClUL9JytmT8o4gMAAAABAAAIwJTEaw0OKd5+IqdDtU6t4mVjaQAriaRz +Oh05ETNUMelerIF8cL9JZ2YurzB0KoDGQTBjb21wb3NlLWhhc2iAUyJvzXtrx+qX +mwpVC/ScrZk/KOI3AWmpUQ5WyQswKdwDAAAAAQAACMCqJbg32BCY1W5ybN/UZk6u +mhJpNiR5dT0Evf0jmkU4igVvJFhBkQkx6gn/gni51II8Z3B1LXBvbGljeS1oYXNo +gEQTb6NVs2eKEUatFvfoZJ6U+0/CH+d+gxDAYPYcqv+KAwAAAAEAAAjAV95k6Lrs +vagFDU9lkiNR6f3lstCMeYm5hwJIaND/XItpm82Rn5lCXz8q2i0vvNHaLGluc3Rh +bmNlLWlkUNr267VFJXjroaE6zR7RAxV7A7/2AwAAAAEAAAjAmL1+a9OVJyC2UCf9 +SUg0BF0GtKcUv3N6Brh0Y4s+oA/0Avf1g+PjsF6SHIVwQzrGMGJvb3QtbXItZG9u +ZQAYQHN5c3RlbS1wcmVwYXJpbmcAGGFwcC1pZFBTIm/Ne2vH6pebClUL9JytmT8o +4jBjb21wb3NlLWhhc2iAUyJvzXtrx+qXmwpVC/ScrZk/KOI3AWmpUQ5WyQswKdw8 +Z3B1LXBvbGljeS1oYXNogEQTb6NVs2eKEUatFvfoZJ6U+0/CH+d+gxDAYPYcqv+K +LGluc3RhbmNlLWlkUNr267VFJXjroaE6zR7RAxV7A7/2MGJvb3QtbXItZG9uZQD+ +uMH/2g9pOTfjREgHn0KeZtYiqMeFgP0zts+/IlUx9jqeGYNYGQuhVzJ4R8SQXdVQ +3MDT45Ys3DNmTEYovHRQkSN7Im9zX2ltYWdlX2hhc2giOiI5OGEzMzk0Njk5NTcw +YWRmNTkyMWViMTdiYWZlZDYyYjM3YjRhNjNjYWQyOGViNTUwNmI0MzBiNjNhZjJl +MjBlIiwiY3B1X2NvdW50IjoyLCJtZW1vcnlfc2l6ZSI6MjE0NzQ4MzY0OCwicWVt +dV92ZXJzaW9uIjoiOC4yLjIiLCJwY2lfaG9sZTY0X3NpemUiOjAsImh1Z2VwYWdl +cyI6ZmFsc2UsIm51bV9ncHVzIjowLCJudW1fbnZzd2l0Y2hlcyI6MCwiaG90cGx1 +Z19vZmYiOmZhbHNlLCJpbWFnZSI6ImRzdGFjay0wLjYuMC12MWF0dGVzdCIsImhv +c3Rfc2hhcmVfbW9kZSI6IjlwIiwib3ZtZl92YXJpYW50IjoicHJlMjAyNTA1Iiwi +dGR4X2F0dGVzdGF0aW9uX3ZhcmlhbnQiOiJsaXRlIiwidGR4X21lYXN1cmVtZW50 +Ijp7ImNoZWNrc3VtX2ZpbGUiOiJOemt3T1dZeU9EazVZV1ZsTldJeE5URmtaVEJs +WVRFM05USTRaR00xTTJabVlqY3lPVGcxTldObFpEVmhOelZoWW1VM1lXTTNNRFUw +WTJNMVpERmpOaUFnYjNadFppNW1aQW95WkdNNVpqbG1NalpoTW1GaVpXWXhOak15 +T0RreE1UUm1OelF6TkRoa056RTNObUprT1RGbVlUZ3hOVGRpWVdVMllqTTJOalJp +TW1WaVpUTm1OakprSUNCaWVrbHRZV2RsQ2pFM01qSTVaamxpTm1FeVpXUmtNalUx +TURJNFpUaGxZamMyTkRRd04yWTJNVGd3TURWa09EQmpZMkV6TkRCak9XWTRPREpt +Wm1JNE5ESTNNbVptTURNZ0lHbHVhWFJ5WVcxbWN5NWpjR2x2TG1kNkNqY3dNRGht +WXpkaE1EUXhZbVJpTjJVNVpqVXhaV0pqT1dRNE9URXlNVEJoTmpVek9HVmtZMkkx +WVRRelpEVTJabUU1WTJZMU1ETTRNemM0Wm1aaE5Ua2dJRzFsZEdGa1lYUmhMbXB6 +YjI0S1lUTXlPRGN4T1dKbVlURXlaRFkwTmpjM05ESmlZalppTm1Gak5ETXhaV1Ex +WkRBeU9UazRZbVpqTkdWbVpqVXpOV1kwWkdVNFpUY3dPR0UyTm1NM01TQWdiV1Zo +YzNWeVpXMWxiblF1ZEdSNExtTmliM0lLWWpjNE1qTTBNekUyTmpZME9URTJOR0pr +WldVeU5ETTVZVFkyTm1ObU9EWTVPVE13WkdNeFpEWTJaV0kxT1RBMU5ESm1NbU0w +TVRJeFlqUTJZalJpTWlBZ2JXVmhjM1Z5WlcxbGJuUXVjMjV3TG1OaWIzSUtaakU0 +Tm1Jek5UQTBZalUyT0dSaVl6WTRNVGsxT1RFNE4yWmlOalJsTWpBMk5UVmxNVGMz +TmpReE5qUTFOekl4WlRjNU5XRXlNekJpTVRGa09HSTBOU0FnYldWaGMzVnlaVzFs +Ym5RdVoyTndMbU5pYjNJS1pUazNPV0psWlRJeE5UazBORFExWWpCaE1UQXpaakZo +TnpZM09EaGhNakUyTXpjd1pUVXdZV1ZtWWpSa016QTVOR1ZrTkRNeU5UVXlNRGc1 +TkdFeVlpQWdiV1ZoYzNWeVpXMWxiblF1WVhkekxtTmliM0lLT0RNellXVTNOalUz +TnpNM04yVTROVEJpTjJNME56UmxNREEwWmpGa1pqSm1OamxqWkdVMVkyTTBZbU13 +WlRKbVlXTmpNemhoWmpBMk5UWXhaRGt4WVNBZ2JXVmhjM1Z5WlcxbGJuUXVZWGR6 +TG5KbGNHeGhlUzVxYzI5dUNnPT0iLCJtZWFzdXJlbWVudCI6Im8yZDJaWEp6YVc5 +dUJHVnBiV0ZuWmFSblkyMWtiR2x1WlhrQkVtTnZibk52YkdVOWRIUjVVekFnYVc1 +cGREMHZhVzVwZENCd1lXNXBZejB4SUc1bGRDNXBabTVoYldWelBUQWdZbWx2YzJS +bGRtNWhiV1U5TUNCdFkyVTliMlptSUc5dmNITTljR0Z1YVdNZ2NHTnBQVzV2WldG +eWJIa2djbUZ1Wkc5dExuUnlkWE4wWDJOd2RUMTVJSEpoYm1SdmJTNTBjblZ6ZEY5 +aWIyOTBiRzloWkdWeVBXNGdkSE5qUFhKbGJHbGhZbXhsSUc1dkxXdDJiV05zYjJO +cklHUnpkR0ZqYXk1eWIyOTBabk5mYUdGemFEMDRORE5rTWpaa1pEYzVNelEwWldO +aE9ERXpZelUzTjJGbVl6ZzVPRFJtWmpGallUYzVORGMzTWpBMVltSmlNMk13WWpC +ak9EUTJPV05oT1RNM1pqSm1JR1J6ZEdGamF5NXliMjkwWm5OZmMybDZaVDAxT1RJ +eU1qUXlOVFp6YTJWeWJtVnNYMkYxZEdobGJuUnBZMjlrWlZndzRyTnBpMFk5MTZM +bW9MRU9XbW93S2pzNXpob1hsUWlyUnNmaU10enB4TjBnVy9VaUtvY1VzbmxzQ1hl +cktkTmNlQmhyWlhKdVpXeGZhR1ZoWkdWeVgyNXZjbTFoYkdsNlpXVDFiV2x1YVhS +eVpGOXphR0V6T0RSWU1QdzBTUUhyOTNJcDF3NkUrVmdpNXY3dHVnT1dINVFKMmtM +clJ5R25LbzdXM2kyWnNGanhiZWdMNytXRjJBemJNR1IwWkhabW8yUnZkbTFtYVhC +eVpUSXdNalV3TldSdGNuUmtvbXR6YVc1bmJHVmZjR0Z6YzFnd0cxeC9nM3ZUdVk2 +Y3BDZTVWaDJVY1Z3NDNuYXZZT2ducGJNWFRCam1SM3FYRWR5VTRoWURpTnRjZEdw +ekJwSVNhSFIzYjE5d1lYTnpXRERIVjZqN0hvTWl0TC9sbjl4TDUwOFRHd0RpT0FZ +cGY3djZoUDB0YjNtb0g3UDUxMEc2dHcrYUNFelgzOUxOK2gxbWRHUmZhRzlpVElB +UUNRUUFCZ2tDQ3dJUUVBPT0ifSwic3BlY192ZXJzaW9uIjoxfTAKBggqhkjOPQQD +AgNJADBGAiEA4YvBDdbKQOpbSvVJbYUh/OujaRDp4qCdmACiBtF630ECIQCAOIkY +4t3tWl45VEVe49a6w334t6jMVusJswHk0wwuuA== +-----END CERTIFICATE----- diff --git a/dstack/verifier/fixtures/tdx-lite-v1-attest.json b/dstack/verifier/fixtures/tdx-lite-v1-attest.json new file mode 100644 index 000000000..72dc9ca7d --- /dev/null +++ b/dstack/verifier/fixtures/tdx-lite-v1-attest.json @@ -0,0 +1,3 @@ +{ + "attestation": "83a776657273696f6e01a8706c6174666f726d82a46b696e64a3746478a46461746182a571756f7465dc138e04000200cc8100000000000000cc93cc9a7233ccf7cc9c4ccca9cc940a0dccb3cc957f0607ccdccca479cc81ccc0ccd1095dcce7cc80ccf4cc91cc950bccc3ccc9000000000e010400000000000000000000000000346bccc77a1846cccaccc214ccdd2ecc8eccdeccb9ccee434944cc9d6c3fcc9fccf2ccc52149cca634ccc27b7fccd1ccbd314c2eccf6ccb973cceeccebccd55742cc952531cca100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000cce702060000000000ccc757cca8ccfb1ecc8322ccb4ccbfcce5cc9fccdc4bcce74f131b00cce23806297fccbbccfacc84ccfd2d6f79cca81fccb3ccf9ccd741ccbaccb70fcc9a084cccd7ccdfccd2cccdccfa1d0153226fcccd7b6bccc7cceacc97cc9b0a550bccf4cc9cccadcc993f28cce2370169cca9510e56ccc90b3029ccdc000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000ccf843cc8dccb36bcc96ccf85dcc8752ccff7f24cca8cc9eccc05c79cceccc9eccda2bcca732ccc8cc97ccfbcc970ccca429365b7471ccb1ccc054cccbcc84ccf17b1c2b23ccba664044cc8ccccaccd93d7ccca8cc9ccc91cc87ccf2ccd0322352ccf8ccea7156cc845a2dccc213ccf44a51ccdcccce52090dcce2cce31eccdeccc55561cce244cc802c71cc8bcce918cce43452cce5ccf7cca96accc108cc9acc84cc8a3159cc902eccc52accf3cc9d4863cc80ccab5b18565e31ccb222ccda46cc99000808ccf2ccad3a0bccc407ccc2cca7ccbaccf33bcca91a734a4eccb4ccec4cccf573ccc3cc8acca4cce66fccd5ccefcc9ccc81131612ccfc6eccf4ccb4cc8fccab230bcc86ccd737cce36453cca1cc9f40cc94cc99cc90cc99cc9811cce5ccfc206b42424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242cccc100000ccc550cc81ccd5ccefcc9cccec7335cc96ccc0cc85ccc76f15662e466eccfdcca6cc97ccae336b78cccdccb271ccaa7b4acce0ccfecc90ccd2cc91ccbf4e10ccd4ccfdcca9ccb32f7f55596279ccf9cce137723c441fccebcc8d7accca2b3eccc0445e655dccfb44cc99ccfb6e3eccb434ccc3275c340dccf9ccefccb8cc96ccc56d62cc8111ccc4ccb955ccc15bccf31774ccc93727636fccdbccbccc8236633bccb83f0838cc8c36cc86cc84ccd3ccc75e29ccaeccd47eccc050cce7720600461000000404191b04ccff0006000000000000000000000000000000000000000000000000000000000000000000000000000000001500000000000000cce700000000000000cce5cca3cca7ccb5ccd830ccc2cc953bcc98534c6c59cca3cca34fccdc34cce933ccf7ccf5cc89cc8f0acc85cccf08cc846bccca0000000000000000000000000000000000000000000000000000000000000000ccdccc9e2a7c6fcc94cc8f17474e34cca7ccfc43cced030f7c1563ccf1ccbaccbdccdf6340ccc82e0e54cca8ccc500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000600000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000ccc209022cccccccc4ccd771ccdc1eccde10ccdb23ccac55ccbe3419ccd22901cca8cc9fccb8cccc6138404337ccb20000000000000000000000000000000000000000000000000000000000000000ccabcc956cccf17851ccd210cc9d63ccacccbfcc911e30cc90ccca1a69cca3cce3485c165f0c2e2f6a36ccb6ccf8cca830ccf755ccd0426a14cc8f022d6a17ccf74500cce6cc8374ccbccc8865ccf3ccf402ccd77c623ecca9ccfbccf62000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f05005e0e00002d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d49494538444343424a65674177494241674956414b576652394d75626c4e5642566e4250454a58614b6466567151394d416f4743437147534d343942414d430a4d484178496a416742674e5642414d4d47556c756447567349464e4857434251513073675547786864475a76636d306751304578476a415942674e5642416f4d0a45556c756447567349454e76636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155450a4341774351304578437a414a42674e5642415954416c56544d423458445449324d4463794e6a41334e4445774f466f5844544d7a4d4463794e6a41334e4445770a4f466f77634445694d434147413155454177775a535735305a5777675530645949464244537942445a584a3061575a70593246305a5445614d426747413155450a43677752535735305a577767513239796347397959585270623234784644415342674e564241634d43314e68626e526849454e7359584a684d517377435159440a5651514944414a445154454c4d416b474131554542684d4356564d775754415442676371686b6a4f5051494242676771686b6a4f50514d4242774e43414151510a644b6d76382f774831684e6476705639474d476345442b514b764f382b6276676f752b426362774c432b79582f78427778384d33395444347262436636764a570a54786f442f496d61784b2b337857554864704d496f3449444444434341776777487759445652306a42426777466f41556c5739647a62306234656c4153636e550a3944504f4156634c336c5177617759445652306642475177596a42676f46366758495a616148523063484d364c79396863476b7564484a316333526c5a484e6c0a636e5a705932567a4c6d6c75644756734c6d4e766253397a5a3367765932567964476c6d61574e6864476c76626939324e4339775932746a636d772f593245390a6347786864475a76636d306d5a57356a62325270626d63395a4756794d42304741315564446751574242515a43536c546e6b372f446b746d702f504a6d304f500a7a417455357a414f42674e56485138424166384542414d434273417744415944565230544151482f4241497741444343416a6b4743537147534962345451454e0a4151534341696f776767496d4d42344743697147534962345451454e415145454547396738357947645143775644587367467967472b6f776767466a42676f710a686b69472b453042445145434d494942557a415142677371686b69472b45304244514543415149424244415142677371686b69472b45304244514543416749420a4244415142677371686b69472b4530424451454341774942416a415142677371686b69472b4530424451454342414942416a415142677371686b69472b4530420a44514543425149424244415142677371686b69472b45304244514543426749424154415142677371686b69472b453042445145434277494241444151426773710a686b69472b45304244514543434149424254415142677371686b69472b45304244514543435149424144415142677371686b69472b45304244514543436749420a4144415142677371686b69472b45304244514543437749424144415142677371686b69472b45304244514543444149424144415142677371686b69472b4530420a44514543445149424144415142677371686b69472b45304244514543446749424144415142677371686b69472b453042445145434477494241444151426773710a686b69472b45304244514543454149424144415142677371686b69472b45304244514543455149424454416642677371686b69472b45304244514543456751510a42415143416751424141554141414141414141414144415142676f71686b69472b45304244514544424149414144415542676f71686b69472b453042445145450a4241615177473841414141774477594b4b6f5a496876684e4151304242516f424154416542676f71686b69472b4530424451454742424238724731754c5a50690a3839584569693555414154684d45514743697147534962345451454e415163774e6a415142677371686b69472b45304244514548415145422f7a4151426773710a686b69472b45304244514548416745422f7a415142677371686b69472b45304244514548417745422f7a414b42676771686b6a4f5051514441674e48414442450a4169414a6d6b77376745482b636d2f4446667275664b3834374f7475546465322f7535615031732f42617237504149675364384c50315161743847506a6a34650a684c2b315470714e64557a2f5946733053515279545941466774343d0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436c6a4343416a32674177494241674956414a567658633239472b487051456e4a3150517a7a674658433935554d416f4743437147534d343942414d430a4d476778476a415942674e5642414d4d45556c756447567349464e48574342536232393049454e424d526f77474159445651514b4442464a626e526c624342440a62334a7762334a6864476c76626a45554d424947413155454277774c553246756447456751327868636d4578437a414a42674e564241674d416b4e424d5173770a435159445651514745774a56557a4165467730784f4441314d6a45784d4455774d5442614677307a4d7a41314d6a45784d4455774d5442614d484178496a41670a42674e5642414d4d47556c756447567349464e4857434251513073675547786864475a76636d306751304578476a415942674e5642416f4d45556c75644756730a49454e76636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b474131554543417743513045780a437a414a42674e5642415954416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a304441516344516741454e53422f377432316c58534f0a3243757a7078773734654a423732457944476757357258437478327456544c7136684b6b367a2b5569525a436e71523770734f766771466553786c6d546c4a6c0a65546d693257597a33714f42757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f536347724442530a42674e5648523845537a424a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b633256790a646d6c6a5a584d75615735305a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e5648513445466751556c5739640a7a62306234656c4153636e553944504f4156634c336c517744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159420a4166384341514177436759494b6f5a497a6a30454177494452774177524149675873566b6930772b6936565947573355462f32327561586530594a446a3155650a6e412b546a44316169356343494359623153416d4435786b66545670766f34556f79695359787244574c6d5552344349394e4b7966504e2b0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d4949436a7a4343416a53674177494241674955496d554d316c71644e496e7a6737535655723951477a6b6e42717777436759494b6f5a497a6a3045417749770a614445614d4267474131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e760a636e4276636d4630615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a0a42674e5642415954416c56544d423458445445344d4455794d5445774e4455784d466f58445451354d54497a4d54497a4e546b314f566f77614445614d4267470a4131554541777752535735305a5777675530645949464a766233516751304578476a415942674e5642416f4d45556c756447567349454e76636e4276636d46300a615739754d5251774567594456515148444174545957353059534244624746795954454c4d416b47413155454341774351304578437a414a42674e56424159540a416c56544d466b77457759484b6f5a497a6a3043415159494b6f5a497a6a3044415163445167414543366e45774d4449595a4f6a2f69505773437a61454b69370a314f694f534c52466857476a626e42564a66566e6b59347533496a6b4459594c304d784f346d717379596a6c42616c54565978465032734a424b357a6c4b4f420a757a43427544416642674e5648534d4547444157674251695a517a575770303069664f44744a5653763141624f5363477244425342674e5648523845537a424a0a4d45656752614244686b466f64485277637a6f764c324e6c636e52705a6d6c6a5958526c63793530636e567a6447566b63325679646d6c6a5a584d75615735300a5a577775593239744c306c756447567355306459556d397664454e424c6d526c636a416442674e564851344546675155496d554d316c71644e496e7a673753560a55723951477a6b6e4271777744675944565230504151482f42415144416745474d42494741315564457745422f7751494d4159424166384341514577436759490a4b6f5a497a6a3045417749445351417752674968414f572f35516b522b533943695344634e6f6f774c7550524c735747662f59693747535839344267775477670a41694541344a306c72486f4d732b586f356f2f7358364f39515778485241765a55474f6452513763767152586171493d0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a96576656e745f6c6f67dc001d85a3696d7200aa6576656e745f74797065ce8000000ba6646967657374c4300b8772e5b0b41b83e6044a68397e02f49fb47066b4fbe4917ea2c45c64f323fdacbb37948f821ebaf8bc9c938ba8a749a56576656e74a0ad6576656e745f7061796c6f6164c42a095464785461626c65000100000000000000af96bb93f2b9b84e9462e0ba74564236009080000000000085a3696d7200aa6576656e745f74797065ce8000000aa6646967657374c430344bc51c980ba621aaa00da3ed7436f7d6e549197dfe699515dfa2c6583d95e6412af21c097d473155875ffd561d6790a56576656e74a0ad6576656e745f7061796c6f6164c43a2946762858585858585858582d585858582d585858582d585858582d58585858585858585858585829000000c0ff00000000004008000000000085a3696d7200aa6576656e745f74797065ce80000001a6646967657374c4309dc3a1f80bcec915391dcda5ffbb15e7419f77eab462bbf72b42166fb70d50325e37b36f93537a863769bcf9bedae6fba56576656e74a0ad6576656e745f7061796c6f6164c43461dfe48bca93d211aa0d00e098032b8c0a00000000000000000000000000000053006500630075007200650042006f006f00740085a3696d7200aa6576656e745f74797065ce80000001a6646967657374c4306f2e3cbc14f9def86980f5f66fd85e99d63e69a73014ed8a5633ce56eca5b64b692108c56110e22acadcef58c3250f1ba56576656e74a0ad6576656e745f7061796c6f6164c42461dfe48bca93d211aa0d00e098032b8c0200000000000000000000000000000050004b0085a3696d7200aa6576656e745f74797065ce80000001a6646967657374c430d607c0efb41c0d757d69bca0615c3a9ac0b1db06c557d992e906c6b7dee40e0e031640c7bfd7bcd35844ef9edeadc6f9a56576656e74a0ad6576656e745f7061796c6f6164c42661dfe48bca93d211aa0d00e098032b8c030000000000000000000000000000004b0045004b0085a3696d7200aa6576656e745f74797065ce80000001a6646967657374c43008a74f8963b337acb6c93682f934496373679dd26af1089cb4eaf0c30cf260a12e814856385ab8843e56a9acea19e127a56576656e74a0ad6576656e745f7061796c6f6164c424cbb219d73a3d9645a3bcdad00e67656f020000000000000000000000000000006400620085a3696d7200aa6576656e745f74797065ce80000001a6646967657374c43018cc6e01f0c6ea99aa23f8a280423e94ad81d96d0aeb5180504fc0f7a40cb3619dd39bd6a95ec1680a86ed6ab0f9828da56576656e74a0ad6576656e745f7061796c6f6164c426cbb219d73a3d9645a3bcdad00e67656f0300000000000000000000000000000064006200780085a3696d7200aa6576656e745f7479706504a6646967657374c430394341b7182cd227c5c6b07ef8000cdfd86136c4292b8e576573ad7ed9ae41019f5818b4b971c9effc60e1ad9f1289f0a56576656e74a0ad6576656e745f7061796c6f6164c4040000000085a3696d7200aa6576656e745f747970650aa6646967657374c430095d04cf26fe03aef6e3561fa24c1aa1cea93f4aeaf563b1f9f7616184c53454875925759434769cec2490acb563a337a56576656e74ab616370692d6c6f61646572ad6576656e745f7061796c6f6164c40941435049204441544185a3696d7200aa6576656e745f747970650aa6646967657374c4308d9a4d4777a1bc77ecd9d8d37a4628129a80052a510320159a20a923bd07a0e90d8d1f2e1ebf088992b25f0d0fa672efa56576656e74a9616370692d72736470ad6576656e745f7061796c6f6164c40941435049204441544185a3696d7200aa6576656e745f747970650aa6646967657374c4303070721e169bc41884724cb0e6b3082e1baf249083d8b389181ba50b9afa951057876c380b8870e8c2facf2eff67a2b6a56576656e74ab616370692d7461626c6573ad6576656e745f7061796c6f6164c40941435049204441544185a3696d7201aa6576656e745f74797065ce80000003a6646967657374c430e2b3698b463dd7a2e6a0b10e5a6a302a3b39ce1a179508ab46c7e232dce9c4dd205bf5222a8714b2796c0977ab29d35ca56576656e74a0ad6576656e745f7061796c6f6164c44a18008a7a0000000000b2c9000000000000000000000000002a000000000000000403140072f728144ab61e44b8c39ebdd7f893c7040412006b00650072006e0065006c0000007fff040085a3696d7200aa6576656e745f74797065ce80000002a6646967657374c4301dd6f7b457ad880d840d41c961283bab688e94e4b59359ea45686581e90feccea3c624b1226113f824f315eb60ae0a7ca56576656e74a0ad6576656e745f7061796c6f6164c43461dfe48bca93d211aa0d00e098032b8c0900000000000000020000000000000042006f006f0074004f007200640065007200000085a3696d7200aa6576656e745f74797065ce80000002a6646967657374c43023ada07f5261f12f34a0bd8e46760962d6b4d576a416f1fea1c64bc656b1d28eacf7047ae6e967c58fd2a98bfa74c298a56576656e74a0ad6576656e745f7061796c6f6164c46e61dfe48bca93d211aa0d00e098032b8c08000000000000003e0000000000000042006f006f0074003000300030003000090100002c0055006900410070007000000004071400c9bdb87cebf8344faaea3ee4af6516a10406140021aa2c4614760345836e8ab6f46623317fff040085a3696d7201aa6576656e745f74797065ce80000007a6646967657374c43077a0dab2312b4e1e57a84d865a21e5b2ee8d677a21012ada819d0a98988078d3d740f6346bfe0abaa938ca20439a8d71a56576656e74a0ad6576656e745f7061796c6f6164c42843616c6c696e6720454649204170706c69636174696f6e2066726f6d20426f6f74204f7074696f6e85a3696d7201aa6576656e745f7479706504a6646967657374c430394341b7182cd227c5c6b07ef8000cdfd86136c4292b8e576573ad7ed9ae41019f5818b4b971c9effc60e1ad9f1289f0a56576656e74a0ad6576656e745f7061796c6f6164c4040000000085a3696d7202aa6576656e745f7479706506a6646967657374c4301fb5131405de515fb00205eb87af9ecf7394ccff92ddcfbe7765bcadc0a829712f045aef33eba215efae2c702b4875dba56576656e74a0ad6576656e745f7061796c6f6164c422ed223b8f1a0000004c4f414445445f494d4147453a3a4c6f61644f7074696f6e730085a3696d7202aa6576656e745f7479706506a6646967657374c430fc344901ebf77229d70e84f95822e6feedba03961f9409da42eb4721a72a8ed6de2d99b058f16de80befe585d80cdb30a56576656e74a0ad6576656e745f7061796c6f6164c415ec223b8f0d0000004c696e757820696e697472640085a3696d7201aa6576656e745f74797065ce80000007a6646967657374c430214b0bef1379756011344877743fdc2a5382bac6e70362d624ccf3f654407c1b4badf7d8f9295dd3dabdef65b27677e0a56576656e74a0ad6576656e745f7061796c6f6164c41d4578697420426f6f7420536572766963657320496e766f636174696f6e85a3696d7201aa6576656e745f74797065ce80000007a6646967657374c4300a2e01c85deae718a530ad8c6d20a84009babe6c8989269e950d8cf440c6e997695e64d455c4174a652cd080f6230b74a56576656e74a0ad6576656e745f7061796c6f6164c4284578697420426f6f742053657276696365732052657475726e65642077697468205375636365737385a3696d7203aa6576656e745f74797065ce08000001a6646967657374c430f9974020ef507068183313d0ca808e0d1ca9b2d1ad0c61f5784e7157c362c06536f5ddacdad4451693f48fcc72fff624a56576656e74b073797374656d2d707265706172696e67ad6576656e745f7061796c6f6164c40085a3696d7203aa6576656e745f74797065ce08000001a6646967657374c430c01f810f36f06cf7f599cf73c5d69c199e61b52892f2c725e847a20593e514e419762ba4546834ffa43564f66154e22ea56576656e74a66170702d6964ad6576656e745f7061796c6f6164c41453226fcd7b6bc7ea979b0a550bf49cad993f28e285a3696d7203aa6576656e745f74797065ce08000001a6646967657374c43094c46b0d0e29de7e22a743b54eade2656369002b89a4733a1d3911335431e95eac817c70bf4967662eaf30742a80c641a56576656e74ac636f6d706f73652d68617368ad6576656e745f7061796c6f6164c42053226fcd7b6bc7ea979b0a550bf49cad993f28e2370169a9510e56c90b3029dc85a3696d7203aa6576656e745f74797065ce08000001a6646967657374c430aa25b837d81098d56e726cdfd4664eae9a1269362479753d04bdfd239a45388a056f245841910931ea09ff8278b9d482a56576656e74af6770752d706f6c6963792d68617368ad6576656e745f7061796c6f6164c42044136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a85a3696d7203aa6576656e745f74797065ce08000001a6646967657374c43057de64e8baecbda8050d4f65922351e9fde5b2d08c7989b987024868d0ff5c8b699bcd919f99425f3f2ada2d2fbcd1daa56576656e74ab696e7374616e63652d6964ad6576656e745f7061796c6f6164c414daf6ebb5452578eba1a13acd1ed103157b03bff685a3696d7203aa6576656e745f74797065ce08000001a6646967657374c43098bd7e6bd3952720b65027fd494834045d06b4a714bf737a06b874638b3ea00ff402f7f583e3e3b05e921c8570433ac6a56576656e74ac626f6f742d6d722d646f6e65ad6576656e745f7061796c6f6164c40085a3696d7203aa6576656e745f74797065ce08000001a6646967657374c4307f788209b6106d43ad4b0cef85797120c234f175a10b7cb609a73ec1f4285adfe7abca2e1e3d3bd2ba058c2b78d1771da56576656e74ac6b65792d70726f7669646572ad6576656e745f7061796c6f6164c4177b226e616d65223a226e6f6e65222c226964223a22227d85a3696d7203aa6576656e745f74797065ce08000001a6646967657374c430ba51104636900268b0e059fa3d266419d079d1e94aea26fb9fcbb8d764bf4c89a67ac271b8a0d1a3989945132a111fc7a56576656e74aa73746f726167652d6673ad6576656e745f7061796c6f6164c4037a667385a3696d7203aa6576656e745f74797065ce08000001a6646967657374c4301a76b2a80a0be71eae59f80945d876351a7a3fb8e9fd1ff1cede5734aa84ea11fd72b4edfbb6f04e5a85edd114c751bda56576656e74ac73797374656d2d7265616479ad6576656e745f7061796c6f6164c400a5737461636b82a46b696e64a664737461636ba46461746183ab7265706f72745f64617461dc004042424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242ae72756e74696d655f6576656e74739982a56576656e74b073797374656d2d707265706172696e67a77061796c6f6164c40082a56576656e74a66170702d6964a77061796c6f6164c41453226fcd7b6bc7ea979b0a550bf49cad993f28e282a56576656e74ac636f6d706f73652d68617368a77061796c6f6164c42053226fcd7b6bc7ea979b0a550bf49cad993f28e2370169a9510e56c90b3029dc82a56576656e74af6770752d706f6c6963792d68617368a77061796c6f6164c42044136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a82a56576656e74ab696e7374616e63652d6964a77061796c6f6164c414daf6ebb5452578eba1a13acd1ed103157b03bff682a56576656e74ac626f6f742d6d722d646f6e65a77061796c6f6164c40082a56576656e74ac6b65792d70726f7669646572a77061796c6f6164c4177b226e616d65223a226e6f6e65222c226964223a22227d82a56576656e74aa73746f726167652d6673a77061796c6f6164c4037a667382a56576656e74ac73797374656d2d7265616479a77061796c6f6164c400a6636f6e666967da08e47b226f735f696d6167655f68617368223a2239386133333934363939353730616466353932316562313762616665643632623337623461363363616432386562353530366234333062363361663265323065222c226370755f636f756e74223a322c226d656d6f72795f73697a65223a323134373438333634382c2271656d755f76657273696f6e223a22382e322e32222c227063695f686f6c6536345f73697a65223a302c22687567657061676573223a66616c73652c226e756d5f67707573223a302c226e756d5f6e767377697463686573223a302c22686f74706c75675f6f6666223a66616c73652c22696d616765223a2264737461636b2d302e362e302d7631617474657374222c22686f73745f73686172655f6d6f6465223a223970222c226f766d665f76617269616e74223a22707265323032353035222c227464785f6174746573746174696f6e5f76617269616e74223a226c697465222c227464785f6d6561737572656d656e74223a7b22636865636b73756d5f66696c65223a224e7a6b774f5759794f446b355957566c4e5749784e54466b5a54426c595445334e5449345a474d314d325a6d596a63794f5467314e574e6c5a4456684e7a5668596d553359574d334d44553059324d315a44466a4e69416762335a745a69356d5a416f795a474d355a6a6c6d4d6a5a684d6d46695a5759784e6a4d794f446b784d54526d4e7a517a4e44686b4e7a45334e6d4a6b4f54466d595467784e54646959575532596a4d324e6a52694d6d56695a544e6d4e6a4a6b49434269656b6c745957646c436a45334d6a49355a6a6c694e6d45795a57526b4d6a55314d4449345a54686c596a63324e4451774e3259324d5467774d44566b4f44426a5932457a4e44426a4f5759344f444a6d5a6d49344e4449334d6d5a6d4d444d6749476c75615852795957316d6379356a63476c764c6d6436436a63774d44686d597a64684d445178596d52694e3255355a6a55785a574a6a4f5751344f5445794d5442684e6a557a4f47566b593249315954517a5a4455325a6d4535593259314d444d344d7a63345a6d5a684e546b674947316c6447466b595852684c6d707a6232344b59544d794f4463784f574a6d595445795a4459304e6a63334e444a69596a5a694e6d466a4e444d785a5751315a4441794f546b34596d5a6a4e47566d5a6a557a4e5759305a4755345a5463774f4745324e6d4d334d53416762575668633356795a57316c626e5175644752344c6d4e696233494b596a63344d6a4d304d7a45324e6a59304f5445324e474a6b5a5755794e444d35595459324e6d4e6d4f4459354f544d775a474d785a4459325a5749314f5441314e444a6d4d6d4d304d544978596a5132596a52694d69416762575668633356795a57316c626e5175633235774c6d4e696233494b5a6a45344e6d497a4e544130596a55324f475269597a59344d546b314f5445344e325a694e6a526c4d6a41324e54566c4d5463334e6a51784e6a51314e7a49785a5463354e5745794d7a42694d54466b4f4749304e53416762575668633356795a57316c626e51755a324e774c6d4e696233494b5a546b334f574a6c5a5449784e546b304e445131596a42684d54417a5a6a46684e7a59334f4468684d6a45324d7a63775a5455775957566d596a526b4d7a41354e47566b4e444d794e5455794d4467354e4745795969416762575668633356795a57316c626e51755958647a4c6d4e696233494b4f444d7a595755334e6a55334e7a4d334e3255344e5442694e324d304e7a526c4d4441305a6a466b5a6a4a6d4e6a6c6a5a47553159324d30596d4d775a544a6d59574e6a4d7a68685a6a41324e5459785a446b785953416762575668633356795a57316c626e51755958647a4c6e4a6c63477868655335716332397543673d3d222c226d6561737572656d656e74223a226f3264325a584a7a61573975424756706257466e5a61526e5932316b62476c755a586b42456d4e76626e4e766247553964485235557a41676157357064443076615735706443427759573570597a30784947356c644335705a6d35686257567a50544167596d6c766332526c646d3568625755394d4342745932553962325a6d4947397663484d396347467561574d6763474e70505735765a57467962486b67636d46755a4739744c6e527964584e3058324e776454313549484a68626d527662533530636e567a6446396962323930624739685a4756795057346764484e6a50584a6c62476c68596d786c494735764c57743262574e7362324e724947527a6447466a61793579623239305a6e4e666147467a614430344e444e6b4d6a5a6b5a4463354d7a51305a574e684f44457a597a55334e32466d597a67354f44526d5a6a466a595463354e4463334d6a4131596d4a694d324d77596a426a4f4451324f574e684f544d335a6a4a6d4947527a6447466a61793579623239305a6e4e6663326c365a5430314f5449794d6a51794e545a7a61325679626d5673583246316447686c626e52705932396b5a56677734724e706930593931364c6d6f4c454f576d6f774b6a73357a686f586c516972527366694d747a70784e3067572f55694b6f6355736e6c73435865724b644e63654268725a584a755a577866614756685a47567958323576636d316862476c365a57543162576c75615852795a46397a6147457a4f4452594d5077305351487239334970317736452b5667693576377475674f574835514a326b4c725279476e4b6f37573369325a73466a786265674c372b574632417a624d4752305a485a6d6f325276646d316d615842795a5449774d6a55774e575274636e526b6f6d747a6157356e624756666347467a633167774731782f6733765475593663704365355668325563567734336e6176594f676e70624d5854426a6d523371584564795534685944694e74636447707a42704953614852336231397759584e7a5744444856366a37486f4d69744c2f6c6e39784c35303854477744694f415970663776366850307462336d6f483750353130473674772b6143457a5833394c4e2b68316d6447526661473969544941514351514142676b434377495145413d3d227d2c22737065635f76657273696f6e223a317d" +} diff --git a/dstack/verifier/fixtures/tdx-lite-v1-issue-cert.pem b/dstack/verifier/fixtures/tdx-lite-v1-issue-cert.pem new file mode 100644 index 000000000..ff5c83558 --- /dev/null +++ b/dstack/verifier/fixtures/tdx-lite-v1-issue-cert.pem @@ -0,0 +1,400 @@ +-----BEGIN CERTIFICATE----- +MIIoBzCCJ66gAwIBAgIUC6Asr0MxljFDBrkXjhFfnsfnFwowCgYIKoZIzj0EAwIw +GDEWMBQGA1UEAwwNQXBwIFJvb3QgQ2VydDAeFw03NTAxMDEwMDAwMDBaFw0zNjA5 +MTEwNTM3MjFaMB4xHDAaBgNVBAMME3YxLWF0dGVzdC1lMmUubG9jYWwwWTATBgcq +hkjOPQIBBggqhkjOPQMBBwNCAAQFNiGH0CuWEOhM1X3LZMTpoWah3fQsk30tqwBP +a7c0x+pxCAiA/dYYALgHScoIWfGfySFCX8+ZoDA4rBQRjhleo4ImzjCCJsowHgYD +VR0RBBcwFYITdjEtYXR0ZXN0LWUyZS5sb2NhbDAPBgNVHQ8BAf8EBQMDB4AAMB0G +A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAkBgorBgEEAYPnPQEDBBYEFFMi +b817a8fql5sKVQv0nK2ZPyjiMBoGCisGAQQBg+c9AQQEDAQKYXBwOmN1c3RvbTCC +JjQGCisGAQQBg+c9AQgEgiYkBIImIIOndmVyc2lvbgGocGxhdGZvcm2CpGtpbmSj +dGR4pGRhdGGCpXF1b3Rl3BOOBAACAMyBAAAAAAAAAMyTzJpyM8z3zJxMzKnMlAoN +zLPMlX8GB8zczKR5zIHMwMzRCV3M58yAzPTMkcyVC8zDzMkAAAAADgEEAAAAAAAA +AAAAAAAAADRrzMd6GEbMyszCFMzdLsyOzN7MuczuQ0lEzJ1sP8yfzPLMxSFJzKY0 +zMJ7f8zRzL0xTC7M9sy5c8zuzOvM1VdCzJUlMcyhAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAMzn +AgYAAAAAAMzHV8yozPsezIMizLTMv8zlzJ/M3EvM508TGwDM4jgGKX/Mu8z6zITM +/S1vecyoH8yzzPnM10HMusy3D8yaCEzM18zfzNLMzcz6HQFTIm/MzXtrzMfM6syX +zJsKVQvM9MyczK3MmT8ozOI3AWnMqVEOVszJCzApzNwAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADM ++EPMjcyza8yWzPhdzIdSzP9/JMyozJ7MwFx5zOzMnszaK8ynMszIzJfM+8yXDMyk +KTZbdHHMsczAVMzLzITM8XscKyPMumZARMyMzMrM2T18zKjMnMyRzIfM8szQMiNS +zPjM6nFWzIRaLczCE8z0SlHM3MzOUgkNzOLM4x7M3szFVWHM4kTMgCxxzIvM6RjM +5DRSzOXM98ypaszBCMyazITMijFZzJAuzMUqzPPMnUhjzIDMq1sYVl4xzLIizNpG +zJkACAjM8sytOgvMxAfMwsynzLrM8zvMqRpzSk7MtMzsTMz1c8zDzIrMpMzmb8zV +zO/MnMyBExYSzPxuzPTMtMyPzKsjC8yGzNc3zONkU8yhzJ9AzJTMmcyQzJnMmBHM +5cz8IGvMwjopzKRbzI90zOsTUMzJacyFzKzM6EU1CMznZcyAzJUoZcyZJGXMqFvM +8lTM1MyxZlM9zN1TzNc8zM8jTR1xzKXMzwdRzJLMwh0OzIvMllI1zIzMsszJzKRq +R2/MzBAAAD1MzJjM28yezM1MYszKLFzMvMypzNzM1AxESMy5MczYF8zLH8yyzKQ3 +VsyozLVATw1czMbMkB/M1cyAeWJRFBAkc8zRByJ7zMzMoifMx8zZzPfM5cysJMy+ +zMvM6MzwfEReZV3M+0TMmcz7bj7MtDTMwydcNA3M+czvzLjMlszFbWLMgRHMxMy5 +VczBW8zzF3TMyTcnY2/M28y8zII2YzvMuD8IOMyMNsyGzITM08zHXinMrszUfszA +UMzncgYARhAAAAQEGRsEzP8ABgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAVAAAAAAAAAMznAAAAAAAAAMzlzKPMp8y1zNgwzMLMlTvMmFNM +bFnMo8yjT8zcNMzpM8z3zPXMicyPCsyFzM8IzIRrzMoAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAMzczJ4qfG/MlMyPF0dONMynzPxDzO0DD3wVY8zxzLrM +vczfY0DMyC4OVMyozMUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAACAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAzMIJAizMzMzEzNdxzNwezN4Q +zNsjzKxVzL40GczSKQHMqMyfzLjMzGE4QEM3zLIAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAMz6RczczNAxzM9RzMhPVzrM8zZazJNlU0DMtMzHTnbM3C9J +zIwAzO7M+sz1zPAEzJo8zLRqA8ziZMzMFsynzP9yLwdfzNrMxXxYzPXM/T8zaMzI +Wcy4zKnM8syxcczUIAAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHwUA +Xg4AAC0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlFOERDQ0JKZWdBd0lC +QWdJVkFLV2ZSOU11YmxOVkJWbkJQRUpYYUtkZlZxUTlNQW9HQ0NxR1NNNDlCQU1D +Ck1IQXhJakFnQmdOVkJBTU1HVWx1ZEdWc0lGTkhXQ0JRUTBzZ1VHeGhkR1p2Y20w +Z1EwRXhHakFZQmdOVkJBb00KRVVsdWRHVnNJRU52Y25CdmNtRjBhVzl1TVJRd0Vn +WURWUVFIREF0VFlXNTBZU0JEYkdGeVlURUxNQWtHQTFVRQpDQXdDUTBFeEN6QUpC +Z05WQkFZVEFsVlRNQjRYRFRJMk1EY3lOakEzTkRFd09Gb1hEVE16TURjeU5qQTNO +REV3Ck9Gb3djREVpTUNBR0ExVUVBd3daU1c1MFpXd2dVMGRZSUZCRFN5QkRaWEow +YVdacFkyRjBaVEVhTUJnR0ExVUUKQ2d3UlNXNTBaV3dnUTI5eWNHOXlZWFJwYjI0 +eEZEQVNCZ05WQkFjTUMxTmhiblJoSUVOc1lYSmhNUXN3Q1FZRApWUVFJREFKRFFU +RUxNQWtHQTFVRUJoTUNWVk13V1RBVEJnY3Foa2pPUFFJQkJnZ3Foa2pPUFFNQkJ3 +TkNBQVFRCmRLbXY4L3dIMWhOZHZwVjlHTUdjRUQrUUt2TzgrYnZnb3UrQmNid0xD +K3lYL3hCd3g4TTM5VEQ0cmJDZjZ2SlcKVHhvRC9JbWF4SyszeFdVSGRwTUlvNElE +RERDQ0F3Z3dId1lEVlIwakJCZ3dGb0FVbFc5ZHpiMGI0ZWxBU2NuVQo5RFBPQVZj +TDNsUXdhd1lEVlIwZkJHUXdZakJnb0Y2Z1hJWmFhSFIwY0hNNkx5OWhjR2t1ZEhK +MWMzUmxaSE5sCmNuWnBZMlZ6TG1sdWRHVnNMbU52YlM5elozZ3ZZMlZ5ZEdsbWFX +TmhkR2x2Ymk5Mk5DOXdZMnRqY213L1kyRTkKY0d4aGRHWnZjbTBtWlc1amIyUnBi +bWM5WkdWeU1CMEdBMVVkRGdRV0JCUVpDU2xUbms3L0RrdG1wL1BKbTBPUAp6QXRV +NXpBT0JnTlZIUThCQWY4RUJBTUNCc0F3REFZRFZSMFRBUUgvQkFJd0FEQ0NBamtH +Q1NxR1NJYjRUUUVOCkFRU0NBaW93Z2dJbU1CNEdDaXFHU0liNFRRRU5BUUVFRUc5 +Zzg1eUdkUUN3VkRYc2dGeWdHK293Z2dGakJnb3EKaGtpRytFMEJEUUVDTUlJQlV6 +QVFCZ3NxaGtpRytFMEJEUUVDQVFJQkJEQVFCZ3NxaGtpRytFMEJEUUVDQWdJQgpC +REFRQmdzcWhraUcrRTBCRFFFQ0F3SUJBakFRQmdzcWhraUcrRTBCRFFFQ0JBSUJB +akFRQmdzcWhraUcrRTBCCkRRRUNCUUlCQkRBUUJnc3Foa2lHK0UwQkRRRUNCZ0lC +QVRBUUJnc3Foa2lHK0UwQkRRRUNCd0lCQURBUUJnc3EKaGtpRytFMEJEUUVDQ0FJ +QkJUQVFCZ3NxaGtpRytFMEJEUUVDQ1FJQkFEQVFCZ3NxaGtpRytFMEJEUUVDQ2dJ +QgpBREFRQmdzcWhraUcrRTBCRFFFQ0N3SUJBREFRQmdzcWhraUcrRTBCRFFFQ0RB +SUJBREFRQmdzcWhraUcrRTBCCkRRRUNEUUlCQURBUUJnc3Foa2lHK0UwQkRRRUNE +Z0lCQURBUUJnc3Foa2lHK0UwQkRRRUNEd0lCQURBUUJnc3EKaGtpRytFMEJEUUVD +RUFJQkFEQVFCZ3NxaGtpRytFMEJEUUVDRVFJQkRUQWZCZ3NxaGtpRytFMEJEUUVD +RWdRUQpCQVFDQWdRQkFBVUFBQUFBQUFBQUFEQVFCZ29xaGtpRytFMEJEUUVEQkFJ +QUFEQVVCZ29xaGtpRytFMEJEUUVFCkJBYVF3RzhBQUFBd0R3WUtLb1pJaHZoTkFR +MEJCUW9CQVRBZUJnb3Foa2lHK0UwQkRRRUdCQkI4ckcxdUxaUGkKODlYRWlpNVVB +QVRoTUVRR0NpcUdTSWI0VFFFTkFRY3dOakFRQmdzcWhraUcrRTBCRFFFSEFRRUIv +ekFRQmdzcQpoa2lHK0UwQkRRRUhBZ0VCL3pBUUJnc3Foa2lHK0UwQkRRRUhBd0VC +L3pBS0JnZ3Foa2pPUFFRREFnTkhBREJFCkFpQUpta3c3Z0VIK2NtL0RGZnJ1Zks4 +NDdPdHVUZGUyL3U1YVAxcy9CYXI3UEFJZ1NkOExQMVFhdDhHUGpqNGUKaEwrMVRw +cU5kVXovWUZzMFNRUnlUWUFGZ3Q0PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t +Ci0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlDbGpDQ0FqMmdBd0lCQWdJ +VkFKVnZYYzI5RytIcFFFbkoxUFF6emdGWEM5NVVNQW9HQ0NxR1NNNDlCQU1DCk1H +Z3hHakFZQmdOVkJBTU1FVWx1ZEdWc0lGTkhXQ0JTYjI5MElFTkJNUm93R0FZRFZR +UUtEQkZKYm5SbGJDQkQKYjNKd2IzSmhkR2x2YmpFVU1CSUdBMVVFQnd3TFUyRnVk +R0VnUTJ4aGNtRXhDekFKQmdOVkJBZ01Ba05CTVFzdwpDUVlEVlFRR0V3SlZVekFl +RncweE9EQTFNakV4TURVd01UQmFGdzB6TXpBMU1qRXhNRFV3TVRCYU1IQXhJakFn +CkJnTlZCQU1NR1VsdWRHVnNJRk5IV0NCUVEwc2dVR3hoZEdadmNtMGdRMEV4R2pB +WUJnTlZCQW9NRVVsdWRHVnMKSUVOdmNuQnZjbUYwYVc5dU1SUXdFZ1lEVlFRSERB +dFRZVzUwWVNCRGJHRnlZVEVMTUFrR0ExVUVDQXdDUTBFeApDekFKQmdOVkJBWVRB +bFZUTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFTlNCLzd0MjFs +WFNPCjJDdXpweHc3NGVKQjcyRXlER2dXNXJYQ3R4MnRWVExxNmhLazZ6K1VpUlpD +bnFSN3BzT3ZncUZlU3hsbVRsSmwKZVRtaTJXWXozcU9CdXpDQnVEQWZCZ05WSFNN +RUdEQVdnQlFpWlF6V1dwMDBpZk9EdEpWU3YxQWJPU2NHckRCUwpCZ05WSFI4RVN6 +QkpNRWVnUmFCRGhrRm9kSFJ3Y3pvdkwyTmxjblJwWm1sallYUmxjeTUwY25WemRH +VmtjMlZ5CmRtbGpaWE11YVc1MFpXd3VZMjl0TDBsdWRHVnNVMGRZVW05dmRFTkJM +bVJsY2pBZEJnTlZIUTRFRmdRVWxXOWQKemIwYjRlbEFTY25VOURQT0FWY0wzbFF3 +RGdZRFZSMFBBUUgvQkFRREFnRUdNQklHQTFVZEV3RUIvd1FJTUFZQgpBZjhDQVFB +d0NnWUlLb1pJemowRUF3SURSd0F3UkFJZ1hzVmtpMHcraTZWWUdXM1VGLzIydWFY +ZTBZSkRqMVVlCm5BK1RqRDFhaTVjQ0lDWWIxU0FtRDV4a2ZUVnB2bzRVb3lpU1l4 +ckRXTG1VUjRDSTlOS3lmUE4rCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0t +LS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNqekNDQWpTZ0F3SUJBZ0lVSW1V +TTFscWROSW56ZzdTVlVyOVFHemtuQnF3d0NnWUlLb1pJemowRUF3SXcKYURFYU1C +Z0dBMVVFQXd3UlNXNTBaV3dnVTBkWUlGSnZiM1FnUTBFeEdqQVlCZ05WQkFvTUVV +bHVkR1ZzSUVOdgpjbkJ2Y21GMGFXOXVNUlF3RWdZRFZRUUhEQXRUWVc1MFlTQkRi +R0Z5WVRFTE1Ba0dBMVVFQ0F3Q1EwRXhDekFKCkJnTlZCQVlUQWxWVE1CNFhEVEU0 +TURVeU1URXdORFV4TUZvWERUUTVNVEl6TVRJek5UazFPVm93YURFYU1CZ0cKQTFV +RUF3d1JTVzUwWld3Z1UwZFlJRkp2YjNRZ1EwRXhHakFZQmdOVkJBb01FVWx1ZEdW +c0lFTnZjbkJ2Y21GMAphVzl1TVJRd0VnWURWUVFIREF0VFlXNTBZU0JEYkdGeVlU +RUxNQWtHQTFVRUNBd0NRMEV4Q3pBSkJnTlZCQVlUCkFsVlRNRmt3RXdZSEtvWkl6 +ajBDQVFZSUtvWkl6ajBEQVFjRFFnQUVDNm5Fd01ESVlaT2ovaVBXc0N6YUVLaTcK +MU9pT1NMUkZoV0dqYm5CVkpmVm5rWTR1M0lqa0RZWUwwTXhPNG1xc3lZamxCYWxU +Vll4RlAyc0pCSzV6bEtPQgp1ekNCdURBZkJnTlZIU01FR0RBV2dCUWlaUXpXV3Aw +MGlmT0R0SlZTdjFBYk9TY0dyREJTQmdOVkhSOEVTekJKCk1FZWdSYUJEaGtGb2RI +Undjem92TDJObGNuUnBabWxqWVhSbGN5NTBjblZ6ZEdWa2MyVnlkbWxqWlhNdWFX +NTAKWld3dVkyOXRMMGx1ZEdWc1UwZFlVbTl2ZEVOQkxtUmxjakFkQmdOVkhRNEVG +Z1FVSW1VTTFscWROSW56ZzdTVgpVcjlRR3prbkJxd3dEZ1lEVlIwUEFRSC9CQVFE +QWdFR01CSUdBMVVkRXdFQi93UUlNQVlCQWY4Q0FRRXdDZ1lJCktvWkl6ajBFQXdJ +RFNRQXdSZ0loQU9XLzVRa1IrUzlDaVNEY05vb3dMdVBSTHNXR2YvWWk3R1NYOTRC +Z3dUd2cKQWlFQTRKMGxySG9NcytYbzVvL3NYNk85UVd4SFJBdlpVR09kUlE3Y3Zx +UlhhcUk9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAACpZXZlbnRfbG9nnIWjaW1yAKpldmVudF90eXBlCqZkaWdlc3TEMAld +BM8m/gOu9uNWH6JMGqHOqT9K6vVjsfn3YWGExTRUh1kldZQ0dpzsJJCstWOjN6Vl +dmVudKthY3BpLWxvYWRlcq1ldmVudF9wYXlsb2FkxAlBQ1BJIERBVEGFo2ltcgCq +ZXZlbnRfdHlwZQqmZGlnZXN0xDCNmk1Hd6G8d+zZ2NN6RigSmoAFKlEDIBWaIKkj +vQeg6Q2NHy4evwiJkrJfDQ+mcu+lZXZlbnSpYWNwaS1yc2RwrWV2ZW50X3BheWxv +YWTECUFDUEkgREFUQYWjaW1yAKpldmVudF90eXBlCqZkaWdlc3TEMDBwch4Wm8QY +hHJMsOazCC4brySQg9iziRgbpQua+pUQV4dsOAuIcOjC+s8u/2eitqVldmVudKth +Y3BpLXRhYmxlc61ldmVudF9wYXlsb2FkxAlBQ1BJIERBVEGFo2ltcgOqZXZlbnRf +dHlwZc4IAAABpmRpZ2VzdMQw+ZdAIO9QcGgYMxPQyoCODRypstGtDGH1eE5xV8Ni +wGU29d2s2tRFFpP0j8xy//YkpWV2ZW50sHN5c3RlbS1wcmVwYXJpbmetZXZlbnRf +cGF5bG9hZMQAhaNpbXIDqmV2ZW50X3R5cGXOCAAAAaZkaWdlc3TEMMAfgQ828Gz3 +9ZnPc8XWnBmeYbUokvLHJehHogWT5RTkGXYrpFRoNP+kNWT2YVTiLqVldmVudKZh +cHAtaWStZXZlbnRfcGF5bG9hZMQUUyJvzXtrx+qXmwpVC/ScrZk/KOKFo2ltcgOq +ZXZlbnRfdHlwZc4IAAABpmRpZ2VzdMQwlMRrDQ4p3n4ip0O1Tq3iZWNpACuJpHM6 +HTkRM1Qx6V6sgXxwv0lnZi6vMHQqgMZBpWV2ZW50rGNvbXBvc2UtaGFzaK1ldmVu +dF9wYXlsb2FkxCBTIm/Ne2vH6pebClUL9JytmT8o4jcBaalRDlbJCzAp3IWjaW1y +A6pldmVudF90eXBlzggAAAGmZGlnZXN0xDCqJbg32BCY1W5ybN/UZk6umhJpNiR5 +dT0Evf0jmkU4igVvJFhBkQkx6gn/gni51IKlZXZlbnSvZ3B1LXBvbGljeS1oYXNo +rWV2ZW50X3BheWxvYWTEIEQTb6NVs2eKEUatFvfoZJ6U+0/CH+d+gxDAYPYcqv+K +haNpbXIDqmV2ZW50X3R5cGXOCAAAAaZkaWdlc3TEMFfeZOi67L2oBQ1PZZIjUen9 +5bLQjHmJuYcCSGjQ/1yLaZvNkZ+ZQl8/KtotL7zR2qVldmVudKtpbnN0YW5jZS1p +ZK1ldmVudF9wYXlsb2FkxBTa9uu1RSV466GhOs0e0QMVewO/9oWjaW1yA6pldmVu +dF90eXBlzggAAAGmZGlnZXN0xDCYvX5r05UnILZQJ/1JSDQEXQa0pxS/c3oGuHRj +iz6gD/QC9/WD4+OwXpIchXBDOsalZXZlbnSsYm9vdC1tci1kb25lrWV2ZW50X3Bh +eWxvYWTEAIWjaW1yA6pldmVudF90eXBlzggAAAGmZGlnZXN0xDB/eIIJthBtQ61L +DO+FeXEgwjTxdaELfLYJpz7B9Cha3+eryi4ePTvSugWMK3jRdx2lZXZlbnSsa2V5 +LXByb3ZpZGVyrWV2ZW50X3BheWxvYWTEF3sibmFtZSI6Im5vbmUiLCJpZCI6IiJ9 +haNpbXIDqmV2ZW50X3R5cGXOCAAAAaZkaWdlc3TEMLpREEY2kAJosOBZ+j0mZBnQ +edHpSuom+5/LuNdkv0yJpnrCcbig0aOYmUUTKhEfx6VldmVudKpzdG9yYWdlLWZz +rWV2ZW50X3BheWxvYWTEA3pmc4WjaW1yA6pldmVudF90eXBlzggAAAGmZGlnZXN0 +xDAadrKoCgvnHq5Z+AlF2HY1Gno/uOn9H/HO3lc0qoTqEf1ytO37tvBOWoXt0RTH +Ub2lZXZlbnSsc3lzdGVtLXJlYWR5rWV2ZW50X3BheWxvYWTEAKVzdGFja4Kka2lu +ZKZkc3RhY2ukZGF0YYOrcmVwb3J0X2RhdGHcAEDMwjopzKRbzI90zOsTUMzJacyF +zKzM6EU1CMznZcyAzJUoZcyZJGXMqFvM8lTM1MyxZlM9zN1TzNc8zM8jTR1xzKXM +zwdRzJLMwh0OzIvMllI1zIzMsszJzKRqR2+ucnVudGltZV9ldmVudHOZgqVldmVu +dLBzeXN0ZW0tcHJlcGFyaW5np3BheWxvYWTEAIKlZXZlbnSmYXBwLWlkp3BheWxv +YWTEFFMib817a8fql5sKVQv0nK2ZPyjigqVldmVudKxjb21wb3NlLWhhc2incGF5 +bG9hZMQgUyJvzXtrx+qXmwpVC/ScrZk/KOI3AWmpUQ5WyQswKdyCpWV2ZW50r2dw +dS1wb2xpY3ktaGFzaKdwYXlsb2FkxCBEE2+jVbNnihFGrRb36GSelPtPwh/nfoMQ +wGD2HKr/ioKlZXZlbnSraW5zdGFuY2UtaWSncGF5bG9hZMQU2vbrtUUleOuhoTrN +HtEDFXsDv/aCpWV2ZW50rGJvb3QtbXItZG9uZadwYXlsb2FkxACCpWV2ZW50rGtl +eS1wcm92aWRlcqdwYXlsb2FkxBd7Im5hbWUiOiJub25lIiwiaWQiOiIifYKlZXZl +bnSqc3RvcmFnZS1mc6dwYXlsb2FkxAN6ZnOCpWV2ZW50rHN5c3RlbS1yZWFkeadw +YXlsb2FkxACmY29uZmln2gjkeyJvc19pbWFnZV9oYXNoIjoiOThhMzM5NDY5OTU3 +MGFkZjU5MjFlYjE3YmFmZWQ2MmIzN2I0YTYzY2FkMjhlYjU1MDZiNDMwYjYzYWYy +ZTIwZSIsImNwdV9jb3VudCI6MiwibWVtb3J5X3NpemUiOjIxNDc0ODM2NDgsInFl +bXVfdmVyc2lvbiI6IjguMi4yIiwicGNpX2hvbGU2NF9zaXplIjowLCJodWdlcGFn +ZXMiOmZhbHNlLCJudW1fZ3B1cyI6MCwibnVtX252c3dpdGNoZXMiOjAsImhvdHBs +dWdfb2ZmIjpmYWxzZSwiaW1hZ2UiOiJkc3RhY2stMC42LjAtdjFhdHRlc3QiLCJo +b3N0X3NoYXJlX21vZGUiOiI5cCIsIm92bWZfdmFyaWFudCI6InByZTIwMjUwNSIs +InRkeF9hdHRlc3RhdGlvbl92YXJpYW50IjoibGl0ZSIsInRkeF9tZWFzdXJlbWVu +dCI6eyJjaGVja3N1bV9maWxlIjoiTnprd09XWXlPRGs1WVdWbE5XSXhOVEZrWlRC +bFlURTNOVEk0WkdNMU0yWm1ZamN5T1RnMU5XTmxaRFZoTnpWaFltVTNZV00zTURV +MFkyTTFaREZqTmlBZ2IzWnRaaTVtWkFveVpHTTVaamxtTWpaaE1tRmlaV1l4TmpN +eU9Ea3hNVFJtTnpRek5EaGtOekUzTm1Ka09URm1ZVGd4TlRkaVlXVTJZak0yTmpS +aU1tVmlaVE5tTmpKa0lDQmlla2x0WVdkbENqRTNNakk1WmpsaU5tRXlaV1JrTWpV +MU1ESTRaVGhsWWpjMk5EUXdOMlkyTVRnd01EVmtPREJqWTJFek5EQmpPV1k0T0RK +bVptSTROREkzTW1abU1ETWdJR2x1YVhSeVlXMW1jeTVqY0dsdkxtZDZDamN3TURo +bVl6ZGhNRFF4WW1SaU4yVTVaalV4WldKak9XUTRPVEV5TVRCaE5qVXpPR1ZrWTJJ +MVlUUXpaRFUyWm1FNVkyWTFNRE00TXpjNFptWmhOVGtnSUcxbGRHRmtZWFJoTG1w +emIyNEtZVE15T0RjeE9XSm1ZVEV5WkRZME5qYzNOREppWWpaaU5tRmpORE14WldR +MVpEQXlPVGs0WW1aak5HVm1aalV6TldZMFpHVTRaVGN3T0dFMk5tTTNNU0FnYldW +aGMzVnlaVzFsYm5RdWRHUjRMbU5pYjNJS1lqYzRNak0wTXpFMk5qWTBPVEUyTkdK +a1pXVXlORE01WVRZMk5tTm1PRFk1T1RNd1pHTXhaRFkyWldJMU9UQTFOREptTW1N +ME1USXhZalEyWWpSaU1pQWdiV1ZoYzNWeVpXMWxiblF1YzI1d0xtTmliM0lLWmpF +NE5tSXpOVEEwWWpVMk9HUmlZelk0TVRrMU9URTROMlppTmpSbE1qQTJOVFZsTVRj +M05qUXhOalExTnpJeFpUYzVOV0V5TXpCaU1URmtPR0kwTlNBZ2JXVmhjM1Z5Wlcx +bGJuUXVaMk53TG1OaWIzSUtaVGszT1dKbFpUSXhOVGswTkRRMVlqQmhNVEF6WmpG +aE56WTNPRGhoTWpFMk16Y3daVFV3WVdWbVlqUmtNekE1TkdWa05ETXlOVFV5TURn +NU5HRXlZaUFnYldWaGMzVnlaVzFsYm5RdVlYZHpMbU5pYjNJS09ETXpZV1UzTmpV +M056TTNOMlU0TlRCaU4yTTBOelJsTURBMFpqRmtaakptTmpsalpHVTFZMk0wWW1N +d1pUSm1ZV05qTXpoaFpqQTJOVFl4WkRreFlTQWdiV1ZoYzNWeVpXMWxiblF1WVhk +ekxuSmxjR3hoZVM1cWMyOXVDZz09IiwibWVhc3VyZW1lbnQiOiJvMmQyWlhKemFX +OXVCR1ZwYldGblphUm5ZMjFrYkdsdVpYa0JFbU52Ym5OdmJHVTlkSFI1VXpBZ2FX +NXBkRDB2YVc1cGRDQndZVzVwWXoweElHNWxkQzVwWm01aGJXVnpQVEFnWW1sdmMy +UmxkbTVoYldVOU1DQnRZMlU5YjJabUlHOXZjSE05Y0dGdWFXTWdjR05wUFc1dlpX +RnliSGtnY21GdVpHOXRMblJ5ZFhOMFgyTndkVDE1SUhKaGJtUnZiUzUwY25WemRG +OWliMjkwYkc5aFpHVnlQVzRnZEhOalBYSmxiR2xoWW14bElHNXZMV3QyYldOc2Iy +TnJJR1J6ZEdGamF5NXliMjkwWm5OZmFHRnphRDA0TkROa01qWmtaRGM1TXpRMFpX +TmhPREV6WXpVM04yRm1Zemc1T0RSbVpqRmpZVGM1TkRjM01qQTFZbUppTTJNd1lq +QmpPRFEyT1dOaE9UTTNaakptSUdSemRHRmpheTV5YjI5MFpuTmZjMmw2WlQwMU9U +SXlNalF5TlRaemEyVnlibVZzWDJGMWRHaGxiblJwWTI5a1pWZ3c0ck5waTBZOTE2 +TG1vTEVPV21vd0tqczV6aG9YbFFpclJzZmlNdHpweE4wZ1cvVWlLb2NVc25sc0NY +ZXJLZE5jZUJoclpYSnVaV3hmYUdWaFpHVnlYMjV2Y20xaGJHbDZaV1QxYldsdWFY +UnlaRjl6YUdFek9EUllNUHcwU1FIcjkzSXAxdzZFK1ZnaTV2N3R1Z09XSDVRSjJr +THJSeUduS283VzNpMlpzRmp4YmVnTDcrV0YyQXpiTUdSMFpIWm1vMlJ2ZG0xbWFY +QnlaVEl3TWpVd05XUnRjblJrb210emFXNW5iR1ZmY0dGemMxZ3dHMXgvZzN2VHVZ +NmNwQ2U1VmgyVWNWdzQzbmF2WU9nbnBiTVhUQmptUjNxWEVkeVU0aFlEaU50Y2RH +cHpCcElTYUhSM2IxOXdZWE56V0RESFY2ajdIb01pdEwvbG45eEw1MDhUR3dEaU9B +WXBmN3Y2aFAwdGIzbW9IN1A1MTBHNnR3K2FDRXpYMzlMTitoMW1kR1JmYUc5aVRJ +QVFDUVFBQmdrQ0N3SVFFQT09In0sInNwZWNfdmVyc2lvbiI6MX0wCgYIKoZIzj0E +AwIDRwAwRAIgF36OTLTMiZGpHcsRpb05huwWMFjEmhtLWugCrFyFkuICIHSQ8Ktq +4e7U3N5dhh+58wW0EAluduY62Y6+LmhlZyi/ +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIiCDCCIa2gAwIBAgIUHLqInDbCs4Sejnc06Isj8lqf50wwCgYIKoZIzj0EAwIw +GDEWMBQGA1UEAwwNQXBwIFJvb3QgQ2VydDAeFw03NTAxMDEwMDAwMDBaFw0zNjA5 +MTEwNTM3MDFaMBgxFjAUBgNVBAMMDUFwcCBSb290IENlcnQwWTATBgcqhkjOPQIB +BggqhkjOPQMBBwNCAAQX2ft1fUcsVG1ySNCRw8384dCUkTO0pqhsk63HhtNEVuJo +FtanJnITtkJ3Hs70XCs4xOeMXiHmY2GThIt+o9dJo4Ig0zCCIM8wDwYDVR0PAQH/ +BAUDAweGADAdBgNVHQ4EFgQUwnjZm59enGH8qUSDLplIxRcLs3QwEgYDVR0TAQH/ +BAgwBgEB/wIBATCCIIcGCisGAQQBg+c9AQgEgiB3BIIgcwAAOU4EAAIAgQAAAAAA +AACTmnIz95xMqZQKDbOVfwYH3KR5gcDRCV3ngPSRlQvDyQAAAAAOAQQAAAAAAAAA +AAAAAAAANGvHehhGysIU3S6O3rnuQ0lEnWw/n/LFIUmmNMJ7f9G9MUwu9rlz7uvV +V0KVJTGhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAQAAAAAOcCBgAAAAAAx1eo+x6DIrS/5Z/cS+dPExsA +4jgGKX+7+oT9LW95qB+z+ddBurcPmghM19/SzfodAVMib817a8fql5sKVQv0nK2Z +PyjiNwFpqVEOVskLMCncAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+EONs2uW+F2HUv9/JKiewFx5 +7J7aK6cyyJf7lwykKTZbdHGxwFTLhPF7HCsjumZARIzK2T18qJyRh/LQMiNS+Opx +VoRaLcIT9EpR3M5SCQ3i4x7exVVh4kSALHGL6RjkNFLl96lqwQiahIoxWZAuxSrz +nUhjgKtbGFZeMbIi2kaZAAgI8q06C8QHwqe68zupZkNAx3O/3NL5/2I79JgBl4bX +DCrywo1BSZTG/9iNGISpfuXu66tOpjgtioUOHIKy/rjB/9oPaTk340RIB59CnmbW +IqjHhYD9M7bPvyJVMfY6nhmDWBkLoVcyeEfEkF3VUNzA0+OWLNwzZkxGKLx0UMwQ +AACY+yzK1tbOk6XtbgicG8Ll6H0NtPMmddtHMQPkC5tY4XUViZmSzK2bVINZ6NJ4 +YZqYDo1nCw4fkZKhEwZVGBohRF5lXftEmftuPrQ0wydcNA3577iWxW1igRHEuVXB +W/MXdMk3J2Nv27yCNmM7uD8IOIw2hoTTx14prtR+wFDncgYARhAAAAQEGRsE/wAG +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUAAAAAAAAA +5wAAAAAAAADlo6e12DDClTuYU0xsWaOjT9w06TP39YmPCoXPCIRrygAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA3J4qfG+UjxdHTjSn/EPtAw98FWPxur3f +Y0DILg5UqMUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAACAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwgkCLMzE13HcHt4Q2yOsVb40GdIpAaif +uMxhOEBDN7IAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPpF3NAxz1HI +T1c68zZak2VTQLTHTnbcL0mMAO769fAEmjy0agPiZMwWp/9yLwdf2sV8WPX9PzNo +yFm4qfKxcdQgAAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fBQBeDgAA +LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUU4RENDQkplZ0F3SUJBZ0lW +QUtXZlI5TXVibE5WQlZuQlBFSlhhS2RmVnFROU1Bb0dDQ3FHU000OUJBTUMKTUhB +eElqQWdCZ05WQkFNTUdVbHVkR1ZzSUZOSFdDQlFRMHNnVUd4aGRHWnZjbTBnUTBF +eEdqQVlCZ05WQkFvTQpFVWx1ZEdWc0lFTnZjbkJ2Y21GMGFXOXVNUlF3RWdZRFZR +UUhEQXRUWVc1MFlTQkRiR0Z5WVRFTE1Ba0dBMVVFCkNBd0NRMEV4Q3pBSkJnTlZC +QVlUQWxWVE1CNFhEVEkyTURjeU5qQTNOREV3T0ZvWERUTXpNRGN5TmpBM05ERXcK +T0Zvd2NERWlNQ0FHQTFVRUF3d1pTVzUwWld3Z1UwZFlJRkJEU3lCRFpYSjBhV1pw +WTJGMFpURWFNQmdHQTFVRQpDZ3dSU1c1MFpXd2dRMjl5Y0c5eVlYUnBiMjR4RkRB +U0JnTlZCQWNNQzFOaGJuUmhJRU5zWVhKaE1Rc3dDUVlEClZRUUlEQUpEUVRFTE1B +a0dBMVVFQmhNQ1ZWTXdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FB +UVEKZEttdjgvd0gxaE5kdnBWOUdNR2NFRCtRS3ZPOCtidmdvdStCY2J3TEMreVgv +eEJ3eDhNMzlURDRyYkNmNnZKVwpUeG9EL0ltYXhLKzN4V1VIZHBNSW80SUREREND +QXdnd0h3WURWUjBqQkJnd0ZvQVVsVzlkemIwYjRlbEFTY25VCjlEUE9BVmNMM2xR +d2F3WURWUjBmQkdRd1lqQmdvRjZnWElaYWFIUjBjSE02THk5aGNHa3VkSEoxYzNS +bFpITmwKY25acFkyVnpMbWx1ZEdWc0xtTnZiUzl6WjNndlkyVnlkR2xtYVdOaGRH +bHZiaTkyTkM5d1kydGpjbXcvWTJFOQpjR3hoZEdadmNtMG1aVzVqYjJScGJtYzla +R1Z5TUIwR0ExVWREZ1FXQkJRWkNTbFRuazcvRGt0bXAvUEptME9QCnpBdFU1ekFP +QmdOVkhROEJBZjhFQkFNQ0JzQXdEQVlEVlIwVEFRSC9CQUl3QURDQ0Fqa0dDU3FH +U0liNFRRRU4KQVFTQ0Fpb3dnZ0ltTUI0R0NpcUdTSWI0VFFFTkFRRUVFRzlnODV5 +R2RRQ3dWRFhzZ0Z5Z0crb3dnZ0ZqQmdvcQpoa2lHK0UwQkRRRUNNSUlCVXpBUUJn +c3Foa2lHK0UwQkRRRUNBUUlCQkRBUUJnc3Foa2lHK0UwQkRRRUNBZ0lCCkJEQVFC +Z3NxaGtpRytFMEJEUUVDQXdJQkFqQVFCZ3NxaGtpRytFMEJEUUVDQkFJQkFqQVFC +Z3NxaGtpRytFMEIKRFFFQ0JRSUJCREFRQmdzcWhraUcrRTBCRFFFQ0JnSUJBVEFR +QmdzcWhraUcrRTBCRFFFQ0J3SUJBREFRQmdzcQpoa2lHK0UwQkRRRUNDQUlCQlRB +UUJnc3Foa2lHK0UwQkRRRUNDUUlCQURBUUJnc3Foa2lHK0UwQkRRRUNDZ0lCCkFE +QVFCZ3NxaGtpRytFMEJEUUVDQ3dJQkFEQVFCZ3NxaGtpRytFMEJEUUVDREFJQkFE +QVFCZ3NxaGtpRytFMEIKRFFFQ0RRSUJBREFRQmdzcWhraUcrRTBCRFFFQ0RnSUJB +REFRQmdzcWhraUcrRTBCRFFFQ0R3SUJBREFRQmdzcQpoa2lHK0UwQkRRRUNFQUlC +QURBUUJnc3Foa2lHK0UwQkRRRUNFUUlCRFRBZkJnc3Foa2lHK0UwQkRRRUNFZ1FR +CkJBUUNBZ1FCQUFVQUFBQUFBQUFBQURBUUJnb3Foa2lHK0UwQkRRRURCQUlBQURB +VUJnb3Foa2lHK0UwQkRRRUUKQkFhUXdHOEFBQUF3RHdZS0tvWklodmhOQVEwQkJR +b0JBVEFlQmdvcWhraUcrRTBCRFFFR0JCQjhyRzF1TFpQaQo4OVhFaWk1VUFBVGhN +RVFHQ2lxR1NJYjRUUUVOQVFjd05qQVFCZ3NxaGtpRytFMEJEUUVIQVFFQi96QVFC +Z3NxCmhraUcrRTBCRFFFSEFnRUIvekFRQmdzcWhraUcrRTBCRFFFSEF3RUIvekFL +QmdncWhrak9QUVFEQWdOSEFEQkUKQWlBSm1rdzdnRUgrY20vREZmcnVmSzg0N090 +dVRkZTIvdTVhUDFzL0JhcjdQQUlnU2Q4TFAxUWF0OEdQamo0ZQpoTCsxVHBxTmRV +ei9ZRnMwU1FSeVRZQUZndDQ9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0t +LS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNsakNDQWoyZ0F3SUJBZ0lWQUpW +dlhjMjlHK0hwUUVuSjFQUXp6Z0ZYQzk1VU1Bb0dDQ3FHU000OUJBTUMKTUdneEdq +QVlCZ05WQkFNTUVVbHVkR1ZzSUZOSFdDQlNiMjkwSUVOQk1Sb3dHQVlEVlFRS0RC +RkpiblJsYkNCRApiM0p3YjNKaGRHbHZiakVVTUJJR0ExVUVCd3dMVTJGdWRHRWdR +MnhoY21FeEN6QUpCZ05WQkFnTUFrTkJNUXN3CkNRWURWUVFHRXdKVlV6QWVGdzB4 +T0RBMU1qRXhNRFV3TVRCYUZ3MHpNekExTWpFeE1EVXdNVEJhTUhBeElqQWcKQmdO +VkJBTU1HVWx1ZEdWc0lGTkhXQ0JRUTBzZ1VHeGhkR1p2Y20wZ1EwRXhHakFZQmdO +VkJBb01FVWx1ZEdWcwpJRU52Y25CdmNtRjBhVzl1TVJRd0VnWURWUVFIREF0VFlX +NTBZU0JEYkdGeVlURUxNQWtHQTFVRUNBd0NRMEV4CkN6QUpCZ05WQkFZVEFsVlRN +Rmt3RXdZSEtvWkl6ajBDQVFZSUtvWkl6ajBEQVFjRFFnQUVOU0IvN3QyMWxYU08K +MkN1enB4dzc0ZUpCNzJFeURHZ1c1clhDdHgydFZUTHE2aEtrNnorVWlSWkNucVI3 +cHNPdmdxRmVTeGxtVGxKbAplVG1pMldZejNxT0J1ekNCdURBZkJnTlZIU01FR0RB +V2dCUWlaUXpXV3AwMGlmT0R0SlZTdjFBYk9TY0dyREJTCkJnTlZIUjhFU3pCSk1F +ZWdSYUJEaGtGb2RIUndjem92TDJObGNuUnBabWxqWVhSbGN5NTBjblZ6ZEdWa2My +VnkKZG1salpYTXVhVzUwWld3dVkyOXRMMGx1ZEdWc1UwZFlVbTl2ZEVOQkxtUmxj +akFkQmdOVkhRNEVGZ1FVbFc5ZAp6YjBiNGVsQVNjblU5RFBPQVZjTDNsUXdEZ1lE +VlIwUEFRSC9CQVFEQWdFR01CSUdBMVVkRXdFQi93UUlNQVlCCkFmOENBUUF3Q2dZ +SUtvWkl6ajBFQXdJRFJ3QXdSQUlnWHNWa2kwdytpNlZZR1czVUYvMjJ1YVhlMFlK +RGoxVWUKbkErVGpEMWFpNWNDSUNZYjFTQW1ENXhrZlRWcHZvNFVveWlTWXhyRFdM +bVVSNENJOU5LeWZQTisKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQotLS0tLUJF +R0lOIENFUlRJRklDQVRFLS0tLS0KTUlJQ2p6Q0NBalNnQXdJQkFnSVVJbVVNMWxx +ZE5JbnpnN1NWVXI5UUd6a25CcXd3Q2dZSUtvWkl6ajBFQXdJdwphREVhTUJnR0Ex +VUVBd3dSU1c1MFpXd2dVMGRZSUZKdmIzUWdRMEV4R2pBWUJnTlZCQW9NRVVsdWRH +VnNJRU52CmNuQnZjbUYwYVc5dU1SUXdFZ1lEVlFRSERBdFRZVzUwWVNCRGJHRnlZ +VEVMTUFrR0ExVUVDQXdDUTBFeEN6QUoKQmdOVkJBWVRBbFZUTUI0WERURTRNRFV5 +TVRFd05EVXhNRm9YRFRRNU1USXpNVEl6TlRrMU9Wb3dhREVhTUJnRwpBMVVFQXd3 +UlNXNTBaV3dnVTBkWUlGSnZiM1FnUTBFeEdqQVlCZ05WQkFvTUVVbHVkR1ZzSUVO +dmNuQnZjbUYwCmFXOXVNUlF3RWdZRFZRUUhEQXRUWVc1MFlTQkRiR0Z5WVRFTE1B +a0dBMVVFQ0F3Q1EwRXhDekFKQmdOVkJBWVQKQWxWVE1Ga3dFd1lIS29aSXpqMENB +UVlJS29aSXpqMERBUWNEUWdBRUM2bkV3TURJWVpPai9pUFdzQ3phRUtpNwoxT2lP +U0xSRmhXR2pibkJWSmZWbmtZNHUzSWprRFlZTDBNeE80bXFzeVlqbEJhbFRWWXhG +UDJzSkJLNXpsS09CCnV6Q0J1REFmQmdOVkhTTUVHREFXZ0JRaVpReldXcDAwaWZP +RHRKVlN2MUFiT1NjR3JEQlNCZ05WSFI4RVN6QkoKTUVlZ1JhQkRoa0ZvZEhSd2N6 +b3ZMMk5sY25ScFptbGpZWFJsY3k1MGNuVnpkR1ZrYzJWeWRtbGpaWE11YVc1MApa +V3d1WTI5dEwwbHVkR1ZzVTBkWVVtOXZkRU5CTG1SbGNqQWRCZ05WSFE0RUZnUVVJ +bVVNMWxxZE5JbnpnN1NWClVyOVFHemtuQnF3d0RnWURWUjBQQVFIL0JBUURBZ0VH +TUJJR0ExVWRFd0VCL3dRSU1BWUJBZjhDQVFFd0NnWUkKS29aSXpqMEVBd0lEU1FB +d1JnSWhBT1cvNVFrUitTOUNpU0RjTm9vd0x1UFJMc1dHZi9ZaTdHU1g5NEJnd1R3 +ZwpBaUVBNEowbHJIb01zK1hvNW8vc1g2TzlRV3hIUkF2WlVHT2RSUTdjdnFSWGFx +ST0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQoAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAACQAAAAACgAAAMAJXQTPJv4DrvbjVh+iTBqhzqk/Sur1Y7H592FhhMU0VIdZ +JXWUNHac7CSQrLVjozcsYWNwaS1sb2FkZXIkQUNQSSBEQVRBAAAAAAoAAADAjZpN +R3ehvHfs2djTekYoEpqABSpRAyAVmiCpI70HoOkNjR8uHr8IiZKyXw0PpnLvJGFj +cGktcnNkcCRBQ1BJIERBVEEAAAAACgAAAMAwcHIeFpvEGIRyTLDmswguG68kkIPY +s4kYG6ULmvqVEFeHbDgLiHDowvrPLv9norYsYWNwaS10YWJsZXMkQUNQSSBEQVRB +AwAAAAEAAAjA+ZdAIO9QcGgYMxPQyoCODRypstGtDGH1eE5xV8NiwGU29d2s2tRF +FpP0j8xy//YkQHN5c3RlbS1wcmVwYXJpbmcAAwAAAAEAAAjAwB+BDzbwbPf1mc9z +xdacGZ5htSiS8scl6EeiBZPlFOQZdiukVGg0/6Q1ZPZhVOIuGGFwcC1pZFBTIm/N +e2vH6pebClUL9JytmT8o4gMAAAABAAAIwJTEaw0OKd5+IqdDtU6t4mVjaQAriaRz +Oh05ETNUMelerIF8cL9JZ2YurzB0KoDGQTBjb21wb3NlLWhhc2iAUyJvzXtrx+qX +mwpVC/ScrZk/KOI3AWmpUQ5WyQswKdwDAAAAAQAACMCqJbg32BCY1W5ybN/UZk6u +mhJpNiR5dT0Evf0jmkU4igVvJFhBkQkx6gn/gni51II8Z3B1LXBvbGljeS1oYXNo +gEQTb6NVs2eKEUatFvfoZJ6U+0/CH+d+gxDAYPYcqv+KAwAAAAEAAAjAV95k6Lrs +vagFDU9lkiNR6f3lstCMeYm5hwJIaND/XItpm82Rn5lCXz8q2i0vvNHaLGluc3Rh +bmNlLWlkUNr267VFJXjroaE6zR7RAxV7A7/2AwAAAAEAAAjAmL1+a9OVJyC2UCf9 +SUg0BF0GtKcUv3N6Brh0Y4s+oA/0Avf1g+PjsF6SHIVwQzrGMGJvb3QtbXItZG9u +ZQAYQHN5c3RlbS1wcmVwYXJpbmcAGGFwcC1pZFBTIm/Ne2vH6pebClUL9JytmT8o +4jBjb21wb3NlLWhhc2iAUyJvzXtrx+qXmwpVC/ScrZk/KOI3AWmpUQ5WyQswKdw8 +Z3B1LXBvbGljeS1oYXNogEQTb6NVs2eKEUatFvfoZJ6U+0/CH+d+gxDAYPYcqv+K +LGluc3RhbmNlLWlkUNr267VFJXjroaE6zR7RAxV7A7/2MGJvb3QtbXItZG9uZQD+ +uMH/2g9pOTfjREgHn0KeZtYiqMeFgP0zts+/IlUx9jqeGYNYGQuhVzJ4R8SQXdVQ +3MDT45Ys3DNmTEYovHRQkSN7Im9zX2ltYWdlX2hhc2giOiI5OGEzMzk0Njk5NTcw +YWRmNTkyMWViMTdiYWZlZDYyYjM3YjRhNjNjYWQyOGViNTUwNmI0MzBiNjNhZjJl +MjBlIiwiY3B1X2NvdW50IjoyLCJtZW1vcnlfc2l6ZSI6MjE0NzQ4MzY0OCwicWVt +dV92ZXJzaW9uIjoiOC4yLjIiLCJwY2lfaG9sZTY0X3NpemUiOjAsImh1Z2VwYWdl +cyI6ZmFsc2UsIm51bV9ncHVzIjowLCJudW1fbnZzd2l0Y2hlcyI6MCwiaG90cGx1 +Z19vZmYiOmZhbHNlLCJpbWFnZSI6ImRzdGFjay0wLjYuMC12MWF0dGVzdCIsImhv +c3Rfc2hhcmVfbW9kZSI6IjlwIiwib3ZtZl92YXJpYW50IjoicHJlMjAyNTA1Iiwi +dGR4X2F0dGVzdGF0aW9uX3ZhcmlhbnQiOiJsaXRlIiwidGR4X21lYXN1cmVtZW50 +Ijp7ImNoZWNrc3VtX2ZpbGUiOiJOemt3T1dZeU9EazVZV1ZsTldJeE5URmtaVEJs +WVRFM05USTRaR00xTTJabVlqY3lPVGcxTldObFpEVmhOelZoWW1VM1lXTTNNRFUw +WTJNMVpERmpOaUFnYjNadFppNW1aQW95WkdNNVpqbG1NalpoTW1GaVpXWXhOak15 +T0RreE1UUm1OelF6TkRoa056RTNObUprT1RGbVlUZ3hOVGRpWVdVMllqTTJOalJp +TW1WaVpUTm1OakprSUNCaWVrbHRZV2RsQ2pFM01qSTVaamxpTm1FeVpXUmtNalUx +TURJNFpUaGxZamMyTkRRd04yWTJNVGd3TURWa09EQmpZMkV6TkRCak9XWTRPREpt +Wm1JNE5ESTNNbVptTURNZ0lHbHVhWFJ5WVcxbWN5NWpjR2x2TG1kNkNqY3dNRGht +WXpkaE1EUXhZbVJpTjJVNVpqVXhaV0pqT1dRNE9URXlNVEJoTmpVek9HVmtZMkkx +WVRRelpEVTJabUU1WTJZMU1ETTRNemM0Wm1aaE5Ua2dJRzFsZEdGa1lYUmhMbXB6 +YjI0S1lUTXlPRGN4T1dKbVlURXlaRFkwTmpjM05ESmlZalppTm1Gak5ETXhaV1Ex +WkRBeU9UazRZbVpqTkdWbVpqVXpOV1kwWkdVNFpUY3dPR0UyTm1NM01TQWdiV1Zo +YzNWeVpXMWxiblF1ZEdSNExtTmliM0lLWWpjNE1qTTBNekUyTmpZME9URTJOR0pr +WldVeU5ETTVZVFkyTm1ObU9EWTVPVE13WkdNeFpEWTJaV0kxT1RBMU5ESm1NbU0w +TVRJeFlqUTJZalJpTWlBZ2JXVmhjM1Z5WlcxbGJuUXVjMjV3TG1OaWIzSUtaakU0 +Tm1Jek5UQTBZalUyT0dSaVl6WTRNVGsxT1RFNE4yWmlOalJsTWpBMk5UVmxNVGMz +TmpReE5qUTFOekl4WlRjNU5XRXlNekJpTVRGa09HSTBOU0FnYldWaGMzVnlaVzFs +Ym5RdVoyTndMbU5pYjNJS1pUazNPV0psWlRJeE5UazBORFExWWpCaE1UQXpaakZo +TnpZM09EaGhNakUyTXpjd1pUVXdZV1ZtWWpSa016QTVOR1ZrTkRNeU5UVXlNRGc1 +TkdFeVlpQWdiV1ZoYzNWeVpXMWxiblF1WVhkekxtTmliM0lLT0RNellXVTNOalUz +TnpNM04yVTROVEJpTjJNME56UmxNREEwWmpGa1pqSm1OamxqWkdVMVkyTTBZbU13 +WlRKbVlXTmpNemhoWmpBMk5UWXhaRGt4WVNBZ2JXVmhjM1Z5WlcxbGJuUXVZWGR6 +TG5KbGNHeGhlUzVxYzI5dUNnPT0iLCJtZWFzdXJlbWVudCI6Im8yZDJaWEp6YVc5 +dUJHVnBiV0ZuWmFSblkyMWtiR2x1WlhrQkVtTnZibk52YkdVOWRIUjVVekFnYVc1 +cGREMHZhVzVwZENCd1lXNXBZejB4SUc1bGRDNXBabTVoYldWelBUQWdZbWx2YzJS +bGRtNWhiV1U5TUNCdFkyVTliMlptSUc5dmNITTljR0Z1YVdNZ2NHTnBQVzV2WldG +eWJIa2djbUZ1Wkc5dExuUnlkWE4wWDJOd2RUMTVJSEpoYm1SdmJTNTBjblZ6ZEY5 +aWIyOTBiRzloWkdWeVBXNGdkSE5qUFhKbGJHbGhZbXhsSUc1dkxXdDJiV05zYjJO +cklHUnpkR0ZqYXk1eWIyOTBabk5mYUdGemFEMDRORE5rTWpaa1pEYzVNelEwWldO +aE9ERXpZelUzTjJGbVl6ZzVPRFJtWmpGallUYzVORGMzTWpBMVltSmlNMk13WWpC +ak9EUTJPV05oT1RNM1pqSm1JR1J6ZEdGamF5NXliMjkwWm5OZmMybDZaVDAxT1RJ +eU1qUXlOVFp6YTJWeWJtVnNYMkYxZEdobGJuUnBZMjlrWlZndzRyTnBpMFk5MTZM +bW9MRU9XbW93S2pzNXpob1hsUWlyUnNmaU10enB4TjBnVy9VaUtvY1VzbmxzQ1hl +cktkTmNlQmhyWlhKdVpXeGZhR1ZoWkdWeVgyNXZjbTFoYkdsNlpXVDFiV2x1YVhS +eVpGOXphR0V6T0RSWU1QdzBTUUhyOTNJcDF3NkUrVmdpNXY3dHVnT1dINVFKMmtM +clJ5R25LbzdXM2kyWnNGanhiZWdMNytXRjJBemJNR1IwWkhabW8yUnZkbTFtYVhC +eVpUSXdNalV3TldSdGNuUmtvbXR6YVc1bmJHVmZjR0Z6YzFnd0cxeC9nM3ZUdVk2 +Y3BDZTVWaDJVY1Z3NDNuYXZZT2ducGJNWFRCam1SM3FYRWR5VTRoWURpTnRjZEdw +ekJwSVNhSFIzYjE5d1lYTnpXRERIVjZqN0hvTWl0TC9sbjl4TDUwOFRHd0RpT0FZ +cGY3djZoUDB0YjNtb0g3UDUxMEc2dHcrYUNFelgzOUxOK2gxbWRHUmZhRzlpVElB +UUNRUUFCZ2tDQ3dJUUVBPT0ifSwic3BlY192ZXJzaW9uIjoxfTAKBggqhkjOPQQD +AgNJADBGAiEA4YvBDdbKQOpbSvVJbYUh/OujaRDp4qCdmACiBtF630ECIQCAOIkY +4t3tWl45VEVe49a6w334t6jMVusJswHk0wwuuA== +-----END CERTIFICATE----- diff --git a/dstack/verifier/fixtures/tdx-lite-v1.README.md b/dstack/verifier/fixtures/tdx-lite-v1.README.md new file mode 100644 index 000000000..af08ddb2b --- /dev/null +++ b/dstack/verifier/fixtures/tdx-lite-v1.README.md @@ -0,0 +1,68 @@ +# TDX lite v1 surface fixtures + +These fixtures pin what the `dstack.guest.v1` surface hands out on real Intel TDX +hardware, next to the frozen v0 surface in the same boot. All four files come from +**one boot** of one CVM: + +| File | Source | Wire form | +| --- | --- | --- | +| `tdx-lite-v1-attest.json` | `POST /v1/Attest` | MessagePack V1 | +| `tdx-lite-v0-attest.json` | `GET /Attest` | legacy SCALE V0 | +| `tdx-lite-v1-issue-cert.pem` | `POST /v1/IssueCert`, `usage_ra_tls` | leaf embeds MessagePack V1 | +| `tdx-lite-v0-get-tls-key.pem` | `POST /GetTlsKey`, `usage_ra_tls` | leaf embeds legacy SCALE V0 | + +Both `Attest` calls request 64 bytes of `0x42` as report data. The certificate +files hold the chain the agent returned: the leaf, then the CVM's local CA +(`App Root Cert`). The private keys were not kept. + +The CVM ran with `event_log_version` 1, so the platform itself produced the +legacy form. The v1 bytes are the guest agent's MessagePack re-encoding +(`AttestationWire::MsgpackV1`), which is exactly the case these fixtures exist +to cover. + +## Environment + +- Image: mkosi `prod` flavor built from git revision `452d04ee8365` (the guest + agent reports `git:452d04ee8365c9d0f456` from `/v1/Version`), + `kernel_header_normalized: true`, + `os_image_hash = 98a3394699570adf5921eb17bafed62b37b4a63cad28eb5506b430b63af2e20e` +- `vm_config.tdx_attestation_variant = "lite"`, 2 vCPUs, 2 GiB, QEMU 8.2.2 +- Key provider `none`, so certificates are signed by the in-CVM local CA and the + CVM needs no KMS or gateway + +## Capture + +The VMM ran locally with `tdx_attestation_variant = "lite"`, and the app compose +carried a `pre_launch_script` that called the guest agent over +`/var/run/dstack.sock` and posted each response to a collector on the host +(`10.0.2.2` under QEMU user networking): + +```bash +RD=$(printf '42%.0s' $(seq 1 64)) +curl --unix-socket /var/run/dstack.sock -X POST -H 'Content-Type: application/json' \ + -d "{\"report_data\":\"$RD\"}" http://localhost/v1/Attest +curl --unix-socket /var/run/dstack.sock "http://localhost/Attest?report_data=$RD" +curl --unix-socket /var/run/dstack.sock -X POST -H 'Content-Type: application/json' \ + -d '{"subject":"v1-attest-e2e.local","alt_names":["v1-attest-e2e.local"],"usage_ra_tls":true,"usage_server_auth":true,"usage_client_auth":true}' \ + http://localhost/v1/IssueCert +curl --unix-socket /var/run/dstack.sock -X POST -H 'Content-Type: application/json' \ + -d '{"subject":"v0-attest-e2e.local","alt_names":["v0-attest-e2e.local"],"usage_ra_tls":true,"usage_server_auth":true,"usage_client_auth":true}' \ + http://localhost/GetTlsKey +``` + +## What the tests assert + +- The two `Attest` results verify through `CvmVerifier` with no image download, + with the quote, event log, OS image hash and ACPI tables all verified, and the + two serialized responses are identical. +- Each leaf chains to its CA and passes `ra_tls::attestation::verify_der`, with + the report data bound to the leaf's own public key. The embedded attestation + passes the same full `CvmVerifier` check, and both certificates name the same + CVM. +- The v1 certificate's attestation is rejected against the v0 leaf's key. + +At capture time the same bytes also verified with the `dstack-attest` +verification path of v0.5.9 and v0.5.11: DCAP quote `UpToDate`, RTMR3 replay, +and identical app identity for the MessagePack and SCALE forms. +`dstack-verifier --verify-cert` does not apply to these leaves, because it +accepts only self-signed RA-TLS certificates. diff --git a/dstack/verifier/fixtures/tdx-lite.README.md b/dstack/verifier/fixtures/tdx-lite.README.md index 01b977e92..d76e22f50 100644 --- a/dstack/verifier/fixtures/tdx-lite.README.md +++ b/dstack/verifier/fixtures/tdx-lite.README.md @@ -3,36 +3,54 @@ This fixture was captured from the local meta-dstack e2e stack using TDX `tdx_attestation_variant = "lite"`. It covers the KMS/verifier path that verifies the OS image from `vm_config.tdx_measurement` (`sha256sum.txt` bytes -plus `measurement.tdx.cbor` bytes), without downloading the image and without -running the QEMU ACPI table helper. +plus `measurement.tdx.cbor` bytes), without downloading the image. The ACPI +tables are regenerated in-process from the VM shape, so no QEMU is involved. Files: - `tdx-lite-attestation.json`: verifier input that mimics the KMS `GetAppKey` flow. It contains a stripped `attestation` whose embedded `vm_config` carries `tdx_measurement`. -- `tdx-lite-getquote.json`: raw guest-agent `GetQuoteResponse` captured - via `GetAttestationForAppKey`, including quote, event log, and vm_config. - TDX `GetQuoteResponse` intentionally omits the `attestation` field to keep - the response compact. +- `tdx-lite-getquote.json`: raw guest-agent `GetQuoteResponse`, including + quote, event log, and vm_config -- the shape `DstackGuest.GetQuote` returns. + `GetQuoteResponse` is Intel TDX only and carries no versioned attestation; + use `Attest` for the platform-adaptive form. -Captured with: +Recaptured for measurement document version 4, which replaced the command-line +digest with the command line itself. The original capture predates that, and a +digest cannot be turned back into the string the new document needs, so these +two files were retaken from a fresh boot rather than re-encoded. + +The image is a pre-normalization one, taken as built: the build system +hardcodes `kernel_header_normalized: true` and fails when the OVMF patch does +not apply, so it can no longer produce the image this pair exercises. Only its +derived `measurement.tdx.cbor`, `sha256sum.txt` and `digest.txt` were +regenerated with the current `dstack-mr`, which is why `os_image_hash` moved. ```bash -E2E_APP_TIMEOUT=900 ./e2e/run.sh up \ - --image-dir images \ - --image dstack-0.6.0 \ - --apps 1 \ - --force \ - --kms-image-verify \ - --kms-no-qemu +# 1. regenerate the derived measurement material in a copy of the image dir +dstack-mr tdx-measurement-cbor "$IMAGE_DIR" > "$IMAGE_DIR/measurement.tdx.cbor" +# then refresh that file's sha256sum.txt line and digest.txt + +# 2. boot it, with an app-compose prelaunch script that calls the guest agent +dstack-vmm -c vmm.toml +vmm-cli.py compose --key-provider none --prelaunch-script capture.sh ... +vmm-cli.py deploy --image "$IMAGE_NAME" --vcpu 2 --memory 2G ... + +# 3. capture.sh, inside the guest +curl -s --unix-socket /var/run/dstack.sock "http://localhost/GetQuote?report_data=$RD" +curl -s --unix-socket /var/run/dstack.sock "http://localhost/Attest?report_data=$RD" ``` +`report_data` is 64 bytes of `0x42`, matching `tests/e2e/attestation`. + Important fixture properties: - `vm_config.tdx_attestation_variant = "lite"` -- `vm_config.memory_size = 2147483648` (2 GiB) -- `vm_config.os_image_hash = e6f5cfec20c02e7b97baa213d0f718020b55e040172d90ccbcb946d56c8b09db` +- `vm_config.memory_size = 2147483648` (2 GiB), `vm_config.cpu_count = 2` +- `vm_config.os_image_hash = 1440bbf5b3a79bcef4bb6013d82d52c12d53fe8d19c21c65fb6b636865179ca1` +- `vm_config.tdx_measurement.measurement` is a version 4 document, and its + `image.kernel_header_normalized` is absent, i.e. false. - `vm_config.tdx_measurement.{checksum_file,measurement}` are JSON base64 byte strings. - The raw top-level `event_log` and stripped attestation keep the three named @@ -61,6 +79,59 @@ dstack-verifier --config verifier-no-download.toml \ --verify verifier/fixtures/tdx-lite-attestation.json ``` -Expected result: `Valid: true`, with quote, event log, and OS image hash all -verified, and `ACPI tables verified: false` because lite mode does not validate -ACPI table contents. +Expected result: `Valid: true`, with quote, event log, OS image hash, and ACPI +tables all verified. The ACPI digests are regenerated in-process from the +fixture's VM shape (2 vCPUs, 2 GiB, QEMU 8.2.2) and must equal the ones the +captured CVM reported. + +## Normalized kernel header fixture + +`tdx-lite-normalized-attestation.json` covers the other kernel measurement. +Images whose OVMF normalizes the Linux setup header declare +`"kernel_header_normalized": true` in `metadata.json`, which is mirrored into +the measurement document, and their RTMR[1] is the plain Authenticode hash of the shipped +`bzImage`. The two fixtures above cover the pre-normalization behavior, which +every image built before that landed still has. + +Both normalized fixtures keep their original captures. Their measurement +documents were re-encoded to version 4 in place: the command line that rebuilt +them was confirmed by reproducing the version 3 digest exactly, and the quote, +event log and every RTMR are untouched, because the measured command line did +not change. `sha256sum.txt` and `os_image_hash` moved with the document. + +`tdx-lite-normalized-qemu-10-2-attestation.json` is the **same image** captured +on **QEMU 10.2.1**, a version that does *not* rewrite the header. The pair is +the point of the whole change: + +| | QEMU 8.2.2 | QEMU 10.2.1 | | +| --- | --- | --- | --- | +| MRTD | `78cb3ad7...` | `8ed3f63f...` | differ -- page-add ordering | +| RTMR0 | `68102e7b...` | `131221a6...` | differ -- generated ACPI tables | +| **RTMR1** | `60db95c7...` | `60db95c7...` | **identical** | +| RTMR2 | `f873ce9b...` | `f873ce9b...` | identical | + +8.2.2 is the load-bearing one: it *does* rewrite the header, so that capture +only passes if the firmware actually undid the rewrite rather than merely +agreeing with itself. 10.2.1 then shows the digest did not move. + +- quote, event log and RTMR3 runtime events come from one boot of that CVM +- the event log replays to exactly the RTMR values in the quote +- `dstack-mr measure --cpu 1 --memory 2G --qemu-version 8.2.2` over that image + directory reproduces MRTD, RTMR[0], RTMR[1] and RTMR[2]. It reads + `kernel_header_normalized` from the image's `metadata.json`, so pointing it at + a pre-normalization image is expected to give a different RTMR[1]: + +``` +MRTD 78cb3ad7...316606fb +RTMR0 68102e7b...f07ffe96 +RTMR1 60db95c7...8f79d200 <- Authenticode SHA-384 of the shipped bzImage +RTMR2 f873ce9b...05793535 +``` + +Captured with `tools/vm-runner` against an image whose `ovmf.fd` carries +`0007-OvmfPkg-QemuKernelLoaderFsDxe-normalize-setup-header.patch`. The CVM ran +with no key provider (`kms_enabled` and `local_key_provider_enabled` both +false), which makes the guest mint temporary app keys and boot all the way +through without any external service, and a `pre_launch_script` handed the quote +and the event log back over `notify-host` -- the host-shared 9p mount is +read-only inside the guest. diff --git a/dstack/verifier/src/main.rs b/dstack/verifier/src/main.rs index c3a171c85..44e29e8ba 100644 --- a/dstack/verifier/src/main.rs +++ b/dstack/verifier/src/main.rs @@ -2,10 +2,11 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::sync::Arc; +use std::{net::IpAddr, path::Path, sync::Arc}; use anyhow::{Context, Result}; use clap::Parser; +use dstack_attest::attestation::AttestationVerifierConfig; use dstack_verifier::{ CvmVerifier, VerificationDetails, VerificationRequest, VerificationResponse, }; @@ -13,6 +14,7 @@ use figment::{ providers::{Env, Format, Toml}, Figment, }; +use ra_tls::attestation::AttestationVerifier; use rocket::{fairing::AdHoc, get, post, serde::json::Json, State}; use serde::{Deserialize, Serialize}; use tracing::{error, info}; @@ -34,15 +36,62 @@ struct Cli { } #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct Config { pub address: String, pub port: u16, pub image_cache_dir: String, - pub pccs_url: Option<String>, + #[serde(default)] + pub attestation: AttestationVerifierConfig, pub image_download_url: String, pub image_download_timeout_secs: u64, } +impl Config { + fn validate(&self) -> Result<()> { + self.address + .parse::<IpAddr>() + .with_context(|| format!("invalid verifier address: {}", self.address))?; + anyhow::ensure!(self.port != 0, "verifier port must not be zero"); + anyhow::ensure!( + !self.image_cache_dir.trim().is_empty(), + "image_cache_dir must not be empty" + ); + anyhow::ensure!( + self.image_download_timeout_secs > 0, + "image_download_timeout_secs must be greater than zero" + ); + anyhow::ensure!( + self.image_download_url.contains("{OS_IMAGE_HASH}"), + "image_download_url must contain {{OS_IMAGE_HASH}}" + ); + anyhow::ensure!( + self.image_download_url.starts_with("http://") + || self.image_download_url.starts_with("https://"), + "image_download_url must use http or https" + ); + let probe_url = self + .image_download_url + .replace("{OS_IMAGE_HASH}", &"00".repeat(32)); + let parsed = reqwest::Url::parse(&probe_url).context("invalid image_download_url")?; + debug_assert!(matches!(parsed.scheme(), "http" | "https")); + Ok(()) + } +} + +fn config_figment(config_path: &Path) -> Figment { + Figment::new() + .merge(Toml::string(include_str!("../dstack-verifier.toml"))) + .merge(Toml::file(config_path)) + .merge(Env::prefixed("DSTACK_VERIFIER_").split("__")) +} + +fn load_config(figment: &Figment) -> Result<Config> { + let config: Config = figment.extract().context("Failed to load configuration")?; + config.validate()?; + Ok(config) +} + #[post("/verify", data = "<request>")] async fn verify_cvm( verifier: &State<Arc<CvmVerifier>>, @@ -69,7 +118,7 @@ fn health() -> Json<serde_json::Value> { })) } -async fn run_oneshot(file_path: &str, config: &Config) -> anyhow::Result<()> { +async fn run_oneshot(file_path: &str, config: &Config) -> anyhow::Result<bool> { use std::fs; info!("Running in oneshot mode for file: {}", file_path); @@ -87,7 +136,7 @@ async fn run_oneshot(file_path: &str, config: &Config) -> anyhow::Result<()> { config.image_cache_dir.clone(), config.image_download_url.clone(), std::time::Duration::from_secs(config.image_download_timeout_secs), - config.pccs_url.clone(), + Arc::new(AttestationVerifier::load(&config.attestation)?), ); // Run verification @@ -107,51 +156,42 @@ async fn run_oneshot(file_path: &str, config: &Config) -> anyhow::Result<()> { })?; info!("Stored verification result at {}", output_path); - // Output results - println!("\n=== Verification Results ==="); - println!("Valid: {}", response.is_valid); - println!("Quote verified: {}", response.details.quote_verified); - println!( - "Event log verified: {}", - response.details.event_log_verified - ); - println!( - "OS image hash verified: {}", - response.details.os_image_hash_verified - ); - println!( - "ACPI tables verified: {}", - response.details.acpi_tables_verified - ); - - if let Some(tcb_status) = &response.details.tcb_status { - println!("TCB status: {}", tcb_status); - } + // Emit exactly one machine-readable document on stdout. + println!("{}", serde_json::to_string(&response)?); + Ok(response.is_valid) +} - if !response.details.advisory_ids.is_empty() { - println!("Advisory IDs: {:?}", response.details.advisory_ids); +fn verify_certificate_profile(cert_der: &[u8]) -> anyhow::Result<()> { + let (_, cert) = x509_parser::parse_x509_certificate(cert_der) + .context("failed to parse X.509 certificate")?; + cert.verify_signature(None) + .context("certificate self-signature verification failed")?; + if !cert.validity().is_valid() { + anyhow::bail!("certificate is outside its validity period"); } - - if let Some(reason) = &response.reason { - println!("Reason: {}", reason); + let key_usage = cert + .key_usage() + .context("failed to decode certificate key usage")? + .context("certificate key usage extension missing")?; + if !key_usage.value.digital_signature() { + anyhow::bail!("certificate key usage does not permit digital signatures"); } - - if let Some(report_data) = &response.details.report_data { - println!("Report data: {}", report_data); + let extended = cert + .extended_key_usage() + .context("failed to decode certificate extended key usage")? + .context("certificate extended key usage extension missing")?; + if !extended.value.server_auth && !extended.value.client_auth { + anyhow::bail!( + "certificate extended key usage permits neither server nor client authentication" + ); } - - if let Some(app_info) = &response.details.app_info { - println!("\n=== App Info ==="); - println!("App ID: {}", hex::encode(&app_info.app_id)); - println!("Instance ID: {}", hex::encode(&app_info.instance_id)); - println!("Compose hash: {}", hex::encode(&app_info.compose_hash)); + let san = cert + .subject_alternative_name() + .context("failed to decode certificate SAN")? + .context("certificate SAN extension missing")?; + if san.value.general_names.is_empty() { + anyhow::bail!("certificate SAN extension is empty"); } - - // Exit with appropriate code - if !response.is_valid { - std::process::exit(1); - } - Ok(()) } @@ -166,27 +206,33 @@ async fn run_cert_oneshot(file_path: &str, config: &Config) -> anyhow::Result<() let cert = fs::read(file_path) .map_err(|e| anyhow::anyhow!("failed to read certificate {}: {}", file_path, e))?; - let verified = if cert.starts_with(b"-----BEGIN") { - ra_tls::attestation::verify_pem(&cert, config.pccs_url.as_deref()).await + let cert_der = if cert.starts_with(b"-----BEGIN") { + let (_, pem) = + x509_parser::pem::parse_x509_pem(&cert).context("failed to parse PEM certificate")?; + pem.contents } else { - ra_tls::attestation::verify_der(&cert, config.pccs_url.as_deref()).await - } - .map_err(|e| anyhow::anyhow!("failed to verify RA-TLS certificate: {:#}", e))?; + cert + }; + verify_certificate_profile(&cert_der)?; - let app_info = verified.attestation.decode_app_info(false).ok(); + let attestation_verifier = Arc::new(AttestationVerifier::load(&config.attestation)?); + let verified = ra_tls::attestation::verify_der(&cert_der, attestation_verifier.as_ref()) + .await + .map_err(|e| anyhow::anyhow!("failed to verify RA-TLS certificate: {:#}", e))?; + + let app_info = verified.decode_app_info(false).ok(); // Bind the reported os_image_hash to the attested boot measurement. For // every platform except TDX legacy this is a self-contained check (no image // download); relying parties should only trust `os_image_hash` when // `os_image_hash_verified` is true. - let os_image_hash_verified = verify_cert_os_image_hash(&verified.attestation, config).await; + let os_image_hash_verified = + verify_cert_os_image_hash(&verified.attestation, config, &attestation_verifier).await; let output = serde_json::json!({ "is_valid": true, "details": { - "attestation_mode": verified.attestation.quote.mode(), + "tee_variant": verified.attestation.quote.variant(), "report_data": hex::encode(verified.attestation.report_data), "public_key_der": hex::encode(&verified.public_key_der), - "app_id_extension": verified.app_id.as_ref().map(hex::encode), - "special_usage": verified.special_usage, "app_info": app_info.map(|info| serde_json::json!({ "app_id": hex::encode(info.app_id), "compose_hash": hex::encode(info.compose_hash), @@ -226,6 +272,7 @@ async fn run_cert_oneshot(file_path: &str, config: &Config) -> anyhow::Result<() async fn verify_cert_os_image_hash( attestation: &ra_tls::attestation::VerifiedAttestation, config: &Config, + attestation_verifier: &Arc<AttestationVerifier>, ) -> bool { use ra_tls::attestation::AttestationQuote; // Only TDX legacy verification downloads the image; skip it here and report @@ -242,7 +289,7 @@ async fn verify_cert_os_image_hash( config.image_cache_dir.clone(), config.image_download_url.clone(), std::time::Duration::from_secs(config.image_download_timeout_secs), - config.pccs_url.clone(), + attestation_verifier.clone(), ); let mut details = VerificationDetails::default(); verifier @@ -256,27 +303,38 @@ async fn main() -> Result<()> { { use tracing_subscriber::{fmt, EnvFilter}; let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); - fmt().with_env_filter(filter).with_ansi(false).init(); + fmt() + .with_env_filter(filter) + .with_ansi(false) + .with_writer(std::io::stderr) + .init(); } let cli = Cli::parse(); - let default_config_str = include_str!("../dstack-verifier.toml"); - - let figment = Figment::from(rocket::Config::default()) - .merge(Toml::string(default_config_str)) - .merge(Toml::file(&cli.config)) - .merge(Env::prefixed("DSTACK_VERIFIER_")); - - let config: Config = figment.extract().context("Failed to load configuration")?; - + let config_path = Path::new(&cli.config); + let config_figment = config_figment(config_path); + let config = load_config(&config_figment)?; // Check for oneshot modes if let Some(file_path) = cli.verify { - if let Err(e) = run_oneshot(&file_path, &config).await { - error!("Oneshot verification failed: {:#}", e); - std::process::exit(1); + let (response, exit_code) = match run_oneshot(&file_path, &config).await { + Ok(is_valid) => (None, if is_valid { 0 } else { 1 }), + Err(error) => { + error!("Oneshot verification failed: {error:#}"); + ( + Some(VerificationResponse { + is_valid: false, + details: VerificationDetails::default(), + reason: Some(format!("Internal error: {error:#}")), + }), + 1, + ) + } + }; + if let Some(response) = response { + println!("{}", serde_json::to_string(&response)?); } - std::process::exit(0); + std::process::exit(exit_code); } if let Some(file_path) = cli.verify_cert { if let Err(e) = run_cert_oneshot(&file_path, &config).await { @@ -290,10 +348,11 @@ async fn main() -> Result<()> { config.image_cache_dir.clone(), config.image_download_url.clone(), std::time::Duration::from_secs(config.image_download_timeout_secs), - config.pccs_url.clone(), + Arc::new(AttestationVerifier::load(&config.attestation)?), )); - rocket::custom(figment) + let rocket_figment = Figment::from(rocket::Config::default()).merge(config_figment); + rocket::custom(rocket_figment) .mount("/", rocket::routes![verify_cvm, health]) .manage(verifier) .attach(AdHoc::on_liftoff("Startup", |_| { @@ -306,3 +365,115 @@ async fn main() -> Result<()> { .map_err(|err| anyhow::anyhow!("launch rocket failed: {err:?}"))?; Ok(()) } + +#[cfg(test)] +mod config_tests { + use super::*; + + fn valid_config(extra: &str) -> String { + format!( + r#"address = "127.0.0.1" +port = 18080 +image_cache_dir = "/tmp/dstack-verifier-config-test" +image_download_url = "http://127.0.0.1:18081/{{OS_IMAGE_HASH}}.tar.gz" +image_download_timeout_secs = 7 +{extra} +"# + ) + } + + #[test] + fn config_file_precedence_validation_and_unknown_field_matrix() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("verifier.toml"); + std::fs::write(&path, valid_config("")).unwrap(); + let loaded = load_config(&config_figment(&path)).unwrap(); + assert_eq!(loaded.address, "127.0.0.1"); + assert_eq!(loaded.port, 18080); + assert_eq!(loaded.image_download_timeout_secs, 7); + + for (name, body, expected) in [ + ( + "zero-port", + valid_config("").replace("port = 18080", "port = 0"), + "port must not be zero", + ), + ( + "empty-cache", + valid_config("").replace("/tmp/dstack-verifier-config-test", ""), + "image_cache_dir must not be empty", + ), + ( + "zero-timeout", + valid_config("").replace("timeout_secs = 7", "timeout_secs = 0"), + "must be greater than zero", + ), + ( + "missing-placeholder", + valid_config("").replace("/{OS_IMAGE_HASH}.tar.gz", "/image.tar.gz"), + "must contain {OS_IMAGE_HASH}", + ), + ( + "invalid-scheme", + valid_config("").replace("http://", "file://"), + "must use http or https", + ), + ( + "unknown-field", + valid_config("unknown_policy = true"), + "unknown field", + ), + ] { + std::fs::write(&path, body).unwrap(); + let error = load_config(&config_figment(&path)).expect_err(name); + assert!(format!("{error:#}").contains(expected), "{name}: {error:#}"); + } + + std::fs::write(&path, valid_config("")).unwrap(); + assert_eq!(load_config(&config_figment(&path)).unwrap().port, 18080); + } +} + +#[cfg(test)] +mod certificate_profile_tests { + use super::*; + use ra_tls::cert::CertRequest; + use rcgen::{KeyPair, PKCS_ECDSA_P256_SHA256}; + use std::time::{Duration, SystemTime}; + + fn certificate(not_before: SystemTime, not_after: SystemTime) -> Vec<u8> { + let key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap(); + let alt_names = vec!["clock-boundary.example.test".to_string()]; + CertRequest::builder() + .key(&key) + .subject("clock-boundary.example.test") + .alt_names(&alt_names) + .usage_server_auth(true) + .not_before(not_before) + .not_after(not_after) + .build() + .self_signed() + .unwrap() + .der() + .to_vec() + } + + #[test] + fn certificate_profile_rejects_expired_and_future_certificates() { + let now = SystemTime::now(); + let hour = Duration::from_secs(60 * 60); + + let expired = certificate(now - hour * 2, now - hour); + let future = certificate(now + hour, now + hour * 2); + let current = certificate(now - hour, now + hour); + + for invalid in [&expired, &future] { + let error = verify_certificate_profile(invalid).unwrap_err(); + assert!( + format!("{error:#}").contains("outside its validity period"), + "unexpected validity diagnostic: {error:#}" + ); + } + verify_certificate_profile(¤t).unwrap(); + } +} diff --git a/dstack/verifier/src/types.rs b/dstack/verifier/src/types.rs index 8b886ae2a..d9d71442b 100644 --- a/dstack/verifier/src/types.rs +++ b/dstack/verifier/src/types.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use dstack_types::KeyProviderInfo; -use ra_tls::attestation::{AppInfo, AttestationMode}; +use ra_tls::attestation::{AppInfo, TeeVariant}; use serde::{Deserialize, Serialize}; use serde_human_bytes as serde_bytes; @@ -32,7 +32,7 @@ pub struct VerificationResponse { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct PolicyBootInfo { - pub attestation_mode: AttestationMode, + pub tee_variant: TeeVariant, #[serde(with = "serde_bytes")] pub mr_aggregated: Vec<u8>, #[serde(with = "serde_bytes")] @@ -55,13 +55,13 @@ pub struct PolicyBootInfo { impl PolicyBootInfo { pub fn from_app_info( - attestation_mode: AttestationMode, + tee_variant: TeeVariant, app_info: &AppInfo, tcb_status: String, advisory_ids: Vec<String>, ) -> Self { Self { - attestation_mode, + tee_variant, mr_aggregated: app_info.mr_aggregated.to_vec(), os_image_hash: app_info.os_image_hash.clone(), mr_system: app_info.mr_system.to_vec(), @@ -90,17 +90,22 @@ pub struct VerificationDetails { pub os_image_hash_verified: bool, /// Indicates that TDX ACPI table contents were verified. /// - /// This is true for the full-image TDX path, where the verifier recomputes - /// ACPI tables and checks the resulting RTMRs against the quote. It remains - /// false for TDX lite, which replays ACPI DATA digests from the event log - /// without validating the table contents. + /// Both dstack TDX paths set this. The full-image path recomputes the + /// tables and checks the resulting RTMRs against the quote. The lite path + /// recomputes the three RTMR0 ACPI DATA digests from the declared VM shape + /// and rejects the attestation when they disagree with the ones the guest + /// reported, then rebuilds RTMR0 from the recomputed digests, so neither + /// path lets host-reported table content reach the expected value. + /// + /// It stays false where the check does not apply: GCP TDX, which measures + /// through the vTPM instead, and the SEV-SNP and Nitro Enclave paths. pub acpi_tables_verified: bool, /// dev vs prod OS image, from metadata.json (bound to os_image_hash). None if not exposed. pub os_image_is_dev: Option<bool>, /// dstack OS version, from the same metadata.json. pub os_image_version: Option<String>, - /// Attestation mode that produced the verified quote. - pub attestation_mode: Option<AttestationMode>, + /// TEE variant that produced the verified quote. + pub tee_variant: Option<TeeVariant>, pub report_data: Option<String>, pub tcb_status: Option<String>, pub advisory_ids: Vec<String>, @@ -157,7 +162,7 @@ pub enum RtmrEventStatus { #[cfg(test)] mod tests { use super::*; - use ra_tls::attestation::AttestationMode; + use ra_tls::attestation::TeeVariant; // the README documents sending either `attestation` or // (`quote` + `event_log` + `vm_config`); every field is optional, so any @@ -203,17 +208,18 @@ mod tests { mr_aggregated: [0x66; 32], os_image_hash: vec![0x77; 32], key_provider_info: br#"{"name":"tpm","id":"aws-test"}"#.to_vec(), + init_script_hashes: Some(Vec::new()), }; let boot_info = PolicyBootInfo::from_app_info( - AttestationMode::DstackAwsNitroTpm, + TeeVariant::DstackAwsNitroTpm, &app_info, String::new(), Vec::new(), ); let encoded = serde_json::to_value(&boot_info).unwrap(); - assert_eq!(encoded["attestationMode"], "dstack-aws-nitro-tpm"); + assert_eq!(encoded["teeVariant"], "dstack-aws-nitro-tpm"); assert_eq!(encoded["tcbStatus"], ""); assert_eq!(encoded["advisoryIds"], serde_json::json!([])); assert!(encoded.get("mrAggregated").unwrap().is_string()); diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 2404ec163..191736726 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -4,8 +4,8 @@ use std::{ ffi::OsStr, - path::{Path, PathBuf}, - sync::OnceLock, + path::{Component, Path, PathBuf}, + sync::Arc, time::Duration, }; @@ -17,14 +17,13 @@ use cc_eventlog::{ }, TdxEvent, }; -use dstack_attest::amd_sev_snp::AmdKdsClient; use dstack_mr::{ tdx::TdxRtmr0AcpiHashes, RtmrLog, RtmrLogs, TdxMeasurementDetails, TdxMeasurements, }; -use dstack_types::VmConfig; +use dstack_types::{TdxAttestationVariant, VmConfig}; use hex_literal::hex; use ra_tls::attestation::{ - AppInfo, Attestation, AttestationQuote, DstackVerifiedReport, NitroPcrs, PlatformEvidence, + AppInfo, Attestation, AttestationQuote, AttestationVerifier, DstackVerifiedReport, NitroPcrs, TpmQuote, VerifiedAttestation, VersionedAttestation, }; use serde::{Deserialize, Serialize}; @@ -38,8 +37,22 @@ use crate::types::{ }; /// Return the canonical TCB status and advisory list used by auth policy. +/// +/// Matched exhaustively on purpose: adding a `DstackVerifiedReport` variant must +/// fail the build here rather than silently inherit another platform's TCB +/// semantics through a catch-all arm. pub fn policy_tcb_fields(attestation: &VerifiedAttestation) -> (String, Vec<String>) { match &attestation.report { + DstackVerifiedReport::DstackTdx(report) => { + (report.status.clone(), report.advisory_ids.clone()) + } + // GCP binds a TPM quote next to the TDX quote, but only the TDX report + // carries a TCB surface, so the TPM report contributes nothing here. + DstackVerifiedReport::DstackGcpTdx { tdx_report, .. } => { + (tdx_report.status.clone(), tdx_report.advisory_ids.clone()) + } + // SNP has no upstream status string; it is derived by comparing the + // report's TCB versions, so read it from `tcb_info` rather than a field. DstackVerifiedReport::DstackAmdSevSnp(report) => ( report.tcb_info.tcb_status().to_string(), report.advisory_ids.clone(), @@ -47,13 +60,10 @@ pub fn policy_tcb_fields(attestation: &VerifiedAttestation) -> (String, Vec<Stri // AWS NitroTPM has no TDX/SNP-style TCB surface; a verified attestation // is normalized to "UpToDate" so the verifier's policy boot info matches // the KMS bootAuth payload and passes the shared "UpToDate" auth gate. - // Other no-TCB platforms (e.g. nitro enclave) stay empty and fail-closed. DstackVerifiedReport::DstackAwsNitroTpm(_) => ("UpToDate".to_string(), Vec::new()), - _ => attestation - .report - .tdx_report() - .map(|report| (report.status.clone(), report.advisory_ids.clone())) - .unwrap_or_default(), + // Other no-TCB platforms (currently Nitro Enclave) stay empty so a + // relying party's UpToDate requirement fails closed. + DstackVerifiedReport::DstackNitroEnclave(_) => (String::new(), Vec::new()), } } @@ -62,7 +72,12 @@ fn policy_boot_info_from_verified_app_info( app_info: &AppInfo, ) -> PolicyBootInfo { let (tcb_status, advisory_ids) = policy_tcb_fields(attestation); - PolicyBootInfo::from_app_info(attestation.quote.mode(), app_info, tcb_status, advisory_ids) + PolicyBootInfo::from_app_info( + attestation.quote.variant(), + app_info, + tcb_status, + advisory_ids, + ) } /// best-effort: None for empty/malformed blobs. @@ -168,6 +183,11 @@ fn collect_rtmr_mismatch( // Bump whenever expected RTMR computation changes so stale entries get ignored. // v3: all supported OVMF measurements use the Pre202505 RTMR[0] layout. +// +// Setup-header normalization did not need a bump: images that predate it are +// measured exactly as before, and normalized images are new images whose +// `os_image_hash` -- part of the `VmConfig` this key hashes -- has never been +// cached. const MEASUREMENT_CACHE_VERSION: u32 = 3; #[derive(Clone, Serialize, Deserialize)] @@ -184,14 +204,14 @@ struct ImagePaths { kernel_cmdline: String, is_dev: bool, version: String, + kernel_header_normalized: bool, } pub struct CvmVerifier { pub image_cache_dir: String, pub download_url: String, pub download_timeout: Duration, - pub pccs_url: Option<String>, - amd_kds_client: OnceLock<Result<AmdKdsClient, String>>, + pub attestation_verifier: Arc<AttestationVerifier>, } impl CvmVerifier { @@ -199,24 +219,13 @@ impl CvmVerifier { image_cache_dir: String, download_url: String, download_timeout: Duration, - pccs_url: Option<String>, + attestation_verifier: Arc<AttestationVerifier>, ) -> Self { Self { image_cache_dir, download_url, download_timeout, - pccs_url, - amd_kds_client: OnceLock::new(), - } - } - - fn amd_kds_client(&self) -> Result<&AmdKdsClient> { - match self - .amd_kds_client - .get_or_init(|| AmdKdsClient::new().map_err(|err| format!("{err:#}"))) - { - Ok(client) => Ok(client), - Err(err) => bail!("failed to create amd sev-snp KDS client: {err}"), + attestation_verifier, } } @@ -310,16 +319,15 @@ impl CvmVerifier { kernel_path: &Path, initrd_path: &Path, kernel_cmdline: &str, + kernel_header_normalized: bool, ) -> Result<TdxMeasurementDetails> { let firmware = fw_path.display().to_string(); let kernel = kernel_path.display().to_string(); let initrd = initrd_path.display().to_string(); - // Prefer the explicit variant the image declared; fall back to parsing - // the version out of the image name for pre-`ovmf_variant` deployments. - let ovmf_variant = vm_config - .ovmf_variant - .unwrap_or_else(|| dstack_mr::ovmf_variant_for_image(vm_config.image.as_deref())); + // Prefer the explicit variant the image declared; pre-`ovmf_variant` + // deployments fall back to the only layout that existed back then. + let ovmf_variant = vm_config.ovmf_variant.unwrap_or_default(); let details = dstack_mr::Machine::builder() .cpu_count(vm_config.cpu_count) @@ -331,6 +339,7 @@ impl CvmVerifier { .root_verity(true) .hotplug_off(vm_config.hotplug_off) .maybe_two_pass_add_pages(vm_config.qemu_single_pass_add_pages) + .normalized_setup_header(kernel_header_normalized) .maybe_pic(vm_config.pic) .maybe_qemu_version(vm_config.qemu_version.clone()) .maybe_pci_hole64_size(if vm_config.pci_hole64_size > 0 { @@ -341,6 +350,8 @@ impl CvmVerifier { .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) .num_nics(vm_config.num_nics) + .num_verity_volumes(vm_config.num_verity_volumes) + .swtpm(vm_config.swtpm) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(ovmf_variant) @@ -358,6 +369,7 @@ impl CvmVerifier { kernel_path: &Path, initrd_path: &Path, kernel_cmdline: &str, + kernel_header_normalized: bool, ) -> Result<TdxMeasurements> { self.compute_measurement_details( vm_config, @@ -365,6 +377,7 @@ impl CvmVerifier { kernel_path, initrd_path, kernel_cmdline, + kernel_header_normalized, ) .map(|details| details.measurements) } @@ -376,6 +389,7 @@ impl CvmVerifier { kernel_path: &Path, initrd_path: &Path, kernel_cmdline: &str, + kernel_header_normalized: bool, ) -> Result<TdxMeasurements> { let cache_key = Self::vm_config_cache_key(vm_config)?; @@ -389,6 +403,7 @@ impl CvmVerifier { kernel_path, initrd_path, kernel_cmdline, + kernel_header_normalized, )?; if let Err(e) = self.store_measurements_in_cache(&cache_key, &measurements) { @@ -421,6 +436,115 @@ impl CvmVerifier { .is_some_and(|digest| digest == expected)) } + /// Mirrors the confinement rule `tar::Entry::unpack_in` applies internally: + /// `..` components, absolute paths, and Windows prefixes escape the + /// extraction root, while `.` components are stripped and are harmless. + /// + /// This duplicates the library check on purpose. `Archive::unpack` discards + /// the `unpack_in` return value, so an escaping member is silently dropped + /// and extraction still reports success; checking here turns that into an + /// error and keeps the boundary from widening if the library's behavior + /// ever changes. It must not be *stricter* than the library, though: + /// rejecting `.` components would reject the `./`-prefixed archives that + /// `tar -czf out.tar.gz .` produces, and roughly a third of the images + /// published on download.dstack.org are packed that way. + fn is_confined_archive_path(path: &Path) -> bool { + path.components() + .all(|component| matches!(component, Component::Normal(_) | Component::CurDir)) + } + + /// A manifest name must be literally a file name, because + /// `prune_unlisted_image_files` matches manifest entries against the + /// `file_name()` of each top-level directory entry, and `sha256sum -c` + /// resolves them relative to the extraction root. + fn is_flat_manifest_name(name: &str) -> bool { + Path::new(name) + .file_name() + .is_some_and(|file_name| file_name == OsStr::new(name)) + } + + fn validate_image_manifest_paths(files_doc: &str) -> Result<()> { + for (line_index, line) in files_doc.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + let mut fields = line.split_whitespace(); + let _digest = fields + .next() + .context("image manifest entry is missing a digest")?; + let name = fields + .next() + .context("image manifest entry is missing a path")?; + if fields.next().is_some() { + bail!("image manifest line {} has extra fields", line_index + 1); + } + if !Self::is_flat_manifest_name(name) { + bail!("image manifest line {} has an unsafe path", line_index + 1); + } + if name == "sha256sum.txt" { + bail!("image manifest must not recursively list sha256sum.txt"); + } + } + Ok(()) + } + + fn extract_image_archive(tarball_path: &Path, extracted_dir: &Path) -> Result<()> { + let file = fs_err::File::open(tarball_path).context("Failed to open image archive")?; + // `MultiGzDecoder`, not `GzDecoder`: the latter stops at the first gzip + // member and reports clean EOF, so a concatenated-member archive would + // extract partially without any error. + let decoder = flate2::read::MultiGzDecoder::new(file); + let mut archive = tar::Archive::new(decoder); + for entry in archive.entries().context("Failed to read image archive")? { + let mut entry = entry.context("Failed to read image archive entry")?; + let path = entry + .path() + .context("Failed to decode image archive path")?; + if !Self::is_confined_archive_path(&path) { + bail!("image archive contains unsafe path {}", path.display()); + } + let entry_type = entry.header().entry_type(); + if !(entry_type.is_file() || entry_type.is_dir()) { + bail!( + "image archive contains unsupported entry {}", + path.display() + ); + } + if !entry + .unpack_in(extracted_dir) + .context("Failed to extract image archive entry")? + { + bail!("image archive entry escaped the extraction root"); + } + } + Ok(()) + } + + fn prune_unlisted_image_files(extracted_dir: &Path, files_doc: &str) -> Result<()> { + let listed_files: Vec<&OsStr> = files_doc + .lines() + .flat_map(|line| line.split_whitespace().nth(1)) + .map(|s| s.as_ref()) + .collect(); + let files = fs_err::read_dir(extracted_dir).context("Failed to read directory")?; + for file in files { + let file = file.context("Failed to read directory entry")?; + let filename = file.file_name(); + // sha256sum.txt is the content-addressed OS identity and is needed + // again when a legacy TDX quote is verified from the cache. + if filename != OsStr::new("sha256sum.txt") + && !listed_files.contains(&filename.as_os_str()) + { + if file.path().is_dir() { + fs_err::remove_dir_all(file.path()).context("Failed to remove directory")?; + } else { + fs_err::remove_file(file.path()).context("Failed to remove file")?; + } + } + } + Ok(()) + } + fn tdx_acpi_hashes_from_event_log(event_log: &[TdxEvent]) -> Result<TdxRtmr0AcpiHashes> { let rtmr0_events = event_log .iter() @@ -470,6 +594,31 @@ impl CvmVerifier { }) } + /// Compare recomputed ACPI digests against the ones the guest reported. + /// + /// RTMR0 alone would already fail on a mismatch, but only with an opaque + /// "MRs do not match": name the offending table here so an operator can + /// tell a tampered table apart from an unexpected VM shape. + fn assert_tdx_acpi_hashes_match( + expected: &TdxRtmr0AcpiHashes, + reported: &TdxRtmr0AcpiHashes, + ) -> Result<()> { + for (name, expected, reported) in [ + (TDX_ACPI_LOADER_EVENT, &expected.loader, &reported.loader), + (TDX_ACPI_RSDP_EVENT, &expected.rsdp, &reported.rsdp), + (TDX_ACPI_TABLES_EVENT, &expected.tables, &reported.tables), + ] { + if expected != reported { + bail!( + "TDX lite {name} digest mismatch: expected {}, reported {}", + hex::encode(expected), + hex::encode(reported) + ); + } + } + Ok(()) + } + /// Helper method to ensure image is downloaded and return image paths async fn ensure_image_downloaded(&self, vm_config: &VmConfig) -> Result<ImagePaths> { let hex_os_image_hash = hex::encode(&vm_config.os_image_hash); @@ -499,7 +648,7 @@ impl CvmVerifier { let fw_path = image_dir.join(&image_info.bios); let kernel_path = image_dir.join(&image_info.kernel); let initrd_path = image_dir.join(&image_info.initrd); - let kernel_cmdline = image_info.cmdline + " initrd=initrd"; + let kernel_cmdline = dstack_mr::tdx::measured_kernel_cmdline(&image_info.cmdline); Ok(ImagePaths { image_dir, @@ -509,6 +658,7 @@ impl CvmVerifier { kernel_cmdline, is_dev: image_info.is_dev, version: image_info.version, + kernel_header_normalized: image_info.kernel_header_normalized, }) } @@ -529,6 +679,7 @@ impl CvmVerifier { &image_paths.kernel_path, &image_paths.initrd_path, &image_paths.kernel_cmdline, + image_paths.kernel_header_normalized, ) } @@ -570,17 +721,11 @@ impl CvmVerifier { let debug = request.debug.unwrap_or(false); let attestation = attestation.into_v1(); - let verified = if matches!(&attestation.platform, PlatformEvidence::SevSnp { .. }) { - attestation - .verify_with_amd_kds_client(self.pccs_url.as_deref(), self.amd_kds_client()?) - .await - } else { - attestation.verify(self.pccs_url.as_deref()).await - }; + let verified = attestation.verify(&self.attestation_verifier).await; let verified_attestation = match verified { Ok(att) => { details.quote_verified = true; - details.attestation_mode = Some(att.quote.mode()); + details.tee_variant = Some(att.quote.variant()); // keep the top-level tcb_status consistent with the // boot_info.tcbStatus fed to the auth policy (notably AWS // NitroTPM, which is normalized to "UpToDate" there). @@ -667,17 +812,33 @@ impl CvmVerifier { AttestationQuote::DstackGcpTdx(quote) => { self.verify_os_image_hash_for_gcp_tdx(&vm_config, "e.tpm_quote)?; } - AttestationQuote::DstackTdx(_) => { - if vm_config.tdx_attestation_variant.is_lite() { - self.verify_os_image_hash_for_dstack_tdx_lite( + // The declared scheme alone selects the path, matched exhaustively + // so a new variant fails the build here instead of taking one. + // + // A `tdx_measurement` document must not pull a `Legacy` boot onto + // the lite path. Both paths now verify the ACPI tables, but they + // disagree on what `os_image_hash` means: legacy requires it to be + // the image digest and recomputes every MR from the downloaded + // image, while lite treats it as `sha256(sha256sum.txt)` and trusts + // the attached document for the image-static material. Images + // attach the document whenever they have it, independent of the + // scheme, so honoring it here would silently move a boot the app + // pinned to `Legacy` onto weaker image-identity checks. + // + // `Lite` without a document is rejected by the lite path itself, + // rather than degraded to a download. + AttestationQuote::DstackTdx(_) => match vm_config.tdx_attestation_variant { + TdxAttestationVariant::Legacy => { + self.verify_os_image_hash_for_dstack_tdx( &vm_config, attestation, debug, details, ) .await?; - } else { - self.verify_os_image_hash_for_dstack_tdx( + } + TdxAttestationVariant::Lite => { + self.verify_os_image_hash_for_dstack_tdx_lite( &vm_config, attestation, debug, @@ -685,7 +846,7 @@ impl CvmVerifier { ) .await?; } - } + }, AttestationQuote::DstackNitroEnclave(_) => { let DstackVerifiedReport::DstackNitroEnclave(report) = &attestation.report else { bail!("internal error: nitro quote without a verified nitro report"); @@ -797,6 +958,7 @@ impl CvmVerifier { &image_paths.kernel_path, &image_paths.initrd_path, &image_paths.kernel_cmdline, + image_paths.kernel_header_normalized, ) .context("Failed to compute expected measurements")?; @@ -815,6 +977,7 @@ impl CvmVerifier { &image_paths.kernel_path, &image_paths.initrd_path, &image_paths.kernel_cmdline, + image_paths.kernel_header_normalized, ) .context("Failed to compute expected measurements")?, None, @@ -843,7 +1006,7 @@ impl CvmVerifier { vm_config: &VmConfig, attestation: &VerifiedAttestation, _debug: bool, - _details: &mut VerificationDetails, + details: &mut VerificationDetails, ) -> Result<()> { let Some(report) = &attestation.report.tdx_report() else { bail!("No TDX report"); @@ -890,12 +1053,26 @@ impl CvmVerifier { // Compute expected measurements. TDX lite keeps the unified image hash // and carries split measurement material; verify it without - // downloading the image or running QEMU-derived ACPI table generators. - // The guest labels the three RTMR0 ACPI DATA events as acpi-loader, - // acpi-rsdp, and acpi-tables before exposing the event log, so the - // verifier does not guess based on event order. - let acpi_hashes = Self::tdx_acpi_hashes_from_event_log(event_log) + // downloading the image. The guest labels the three RTMR0 ACPI DATA + // events as acpi-loader, acpi-rsdp, and acpi-tables before exposing the + // event log, so the verifier does not guess based on event order. + let reported_acpi_hashes = Self::tdx_acpi_hashes_from_event_log(event_log) .context("TDX lite attestation is missing named RTMR0 ACPI DATA digests")?; + // Recompute the digests from the declared VM shape instead of trusting + // the reported ones, and treat both failure modes as fatal: a mismatch + // means the tables are not the ones this shape produces, and a shape + // the generator cannot model (`swtpm`, a QEMU older than any profile) + // leaves nothing to compare against. Downgrading either case to "pass, + // but unverified" would make the check optional at the host's + // discretion, because `swtpm` and `qemu_version` are host-declared + // fields that no other measurement independently constrains. + let acpi_hashes = + dstack_mr::tdx::expected_rtmr0_acpi_hashes(vm_config, measurement.tdvf.ovmf_variant) + .context("failed to recompute expected TDX lite ACPI table digests")?; + Self::assert_tdx_acpi_hashes_match(&acpi_hashes, &reported_acpi_hashes)?; + details.acpi_tables_verified = true; + // RTMR0 below is rebuilt from the recomputed digests, so the expected + // value depends on nothing the host reported about the tables. let mrs = dstack_mr::tdx::tdx_measurements_from_measurement_document( document, vm_config, @@ -1144,21 +1321,16 @@ impl CvmVerifier { let extracted_dir = tmp_dir.join("extracted"); fs_err::create_dir_all(&extracted_dir).context("Failed to create extraction directory")?; - // Extract the tarball - let output = Command::new("tar") - .arg("xzf") - .arg(&tarball_path) - .current_dir(&extracted_dir) - .output() + file.flush() .await - .context("Failed to extract tarball")?; + .context("Failed to flush image archive")?; + drop(file); + Self::extract_image_archive(&tarball_path, &extracted_dir)?; - if !output.status.success() { - bail!( - "Failed to extract tarball: {}", - String::from_utf8_lossy(&output.stderr) - ); - } + let sha256sum_path = extracted_dir.join("sha256sum.txt"); + let files_doc = + fs_err::read_to_string(&sha256sum_path).context("Failed to read sha256sum.txt")?; + Self::validate_image_manifest_paths(&files_doc)?; // Verify checksum let output = Command::new("sha256sum") @@ -1177,26 +1349,7 @@ impl CvmVerifier { } // Remove the files that are not listed in sha256sum.txt - let sha256sum_path = extracted_dir.join("sha256sum.txt"); - let files_doc = - fs_err::read_to_string(&sha256sum_path).context("Failed to read sha256sum.txt")?; - let listed_files: Vec<&OsStr> = files_doc - .lines() - .flat_map(|line| line.split_whitespace().nth(1)) - .map(|s| s.as_ref()) - .collect(); - let files = fs_err::read_dir(&extracted_dir).context("Failed to read directory")?; - for file in files { - let file = file.context("Failed to read directory entry")?; - let filename = file.file_name(); - if !listed_files.contains(&filename.as_os_str()) { - if file.path().is_dir() { - fs_err::remove_dir_all(file.path()).context("Failed to remove directory")?; - } else { - fs_err::remove_file(file.path()).context("Failed to remove file")?; - } - } - } + Self::prune_unlisted_image_files(&extracted_dir, &files_doc)?; // All image modes are addressed by sha256(sha256sum.txt). Extra // measurement CBOR files are ordinary sha256sum.txt entries and do not @@ -1272,6 +1425,19 @@ mod tests { use super::*; + // Kept inside `mod tests` so the non-test build stays free of unused imports. + use dcap_qvl::{ + quote::{Report as DcapReport, TDReport10}, + tcb_info::{TcbStatus, TcbStatusWithAdvisory}, + verify::VerifiedReport as DcapVerifiedReport, + }; + use dstack_attest::amd_sev_snp::{AmdSnpTcbInfo, VerifiedAmdSnpReport}; + use ra_tls::attestation::{ + AwsNitroTpmVerifiedReport, DstackAwsNitroTpmQuote, DstackGcpTdxQuote, DstackNitroQuote, + NitroVerifiedReport, SnpQuote, TdxQuote, + }; + use tpm_qvl::verify::{ClockInfo, QuoteInfo, TpmAttest, VerifiedReport as TpmVerifiedReport}; + fn aws_boot_pcrs(pcr4: u8) -> BTreeMap<u16, Vec<u8>> { BTreeMap::from([ (4, vec![pcr4; 48]), @@ -1302,7 +1468,356 @@ mod tests { } fn test_verifier() -> CvmVerifier { - CvmVerifier::new(String::new(), String::new(), Duration::from_secs(1), None) + CvmVerifier::new( + String::new(), + String::new(), + Duration::from_secs(1), + test_attestation_verifier(), + ) + } + + fn test_attestation_verifier() -> Arc<AttestationVerifier> { + Arc::new(AttestationVerifier::new_prod(None).unwrap()) + } + + /// Build a TDX verified report carrying `status`/`advisory_ids`. Everything + /// else is zeroed: `policy_tcb_fields` reads only those two fields. + fn tdx_report(status: &str, advisory_ids: &[&str]) -> DcapVerifiedReport { + DcapVerifiedReport { + status: status.to_string(), + advisory_ids: advisory_ids.iter().map(|id| id.to_string()).collect(), + report: DcapReport::TD10(TDReport10 { + tee_tcb_svn: [0; 16], + mr_seam: [0; 48], + mr_signer_seam: [0; 48], + seam_attributes: [0; 8], + td_attributes: [0; 8], + xfam: [0; 8], + mr_td: [0; 48], + mr_config_id: [0; 48], + mr_owner: [0; 48], + mr_owner_config: [0; 48], + rt_mr0: [0; 48], + rt_mr1: [0; 48], + rt_mr2: [0; 48], + rt_mr3: [0; 48], + report_data: [0; 64], + }), + ppid: Vec::new(), + qe_status: TcbStatusWithAdvisory::new(TcbStatus::UpToDate, Vec::new()), + platform_status: TcbStatusWithAdvisory::new(TcbStatus::UpToDate, Vec::new()), + } + } + + fn snp_report(tcb_info: AmdSnpTcbInfo, advisory_ids: &[&str]) -> VerifiedAmdSnpReport { + VerifiedAmdSnpReport { + measurement: [0; 48], + report_data: [0; 64], + host_data: [0; 32], + chip_id: [0; 64], + tcb_info, + advisory_ids: advisory_ids.iter().map(|id| id.to_string()).collect(), + } + } + + fn tpm_report_for_gcp() -> TpmVerifiedReport { + TpmVerifiedReport { + attest: TpmAttest { + magic: 0, + type_: 0, + qualified_signer: Vec::new(), + qualified_data: Vec::new(), + clock_info: ClockInfo { + clock: 0, + reset_count: 0, + restart_count: 0, + safe: 0, + }, + firmware_version: 0, + attested_quote_info: QuoteInfo { + pcr_selections: Vec::new(), + pcr_digest: Vec::new(), + }, + }, + platform: dstack_types::Platform::Gcp, + pcr_values: Vec::new(), + } + } + + /// `policy_tcb_fields` ignores the quote, but pair each report with its own + /// platform's quote so the rows stay readable as real attestations. + fn verified_attestation( + quote: AttestationQuote, + report: DstackVerifiedReport, + ) -> VerifiedAttestation { + Attestation { + quote, + runtime_events: Vec::new(), + report_data: [0; 64], + config: String::new(), + report, + } + } + + fn empty_tdx_quote() -> AttestationQuote { + AttestationQuote::DstackTdx(TdxQuote { + quote: Vec::new(), + event_log: Vec::new(), + }) + } + + /// Exercises the report-to-policy mapping itself, which is where a wrong + /// source can silently downgrade the auth gate (e.g. reading the wrong + /// report on GCP, or letting a no-TCB platform report "UpToDate"). + #[test] + fn tcb_policy_fields_map_each_platform_to_its_own_tcb_source() { + let advisories = ["INTEL-SA-00001", "INTEL-SA-00002"]; + let expected_advisories: Vec<String> = advisories.iter().map(|id| id.to_string()).collect(); + + // SNP derives its status by comparing TCB versions rather than reading + // a field, so cover both the equal (UpToDate) and unequal (OutOfDate) + // shapes. + let snp_up_to_date = AmdSnpTcbInfo::default(); + let mut snp_out_of_date = AmdSnpTcbInfo::default(); + snp_out_of_date.current.microcode = 1; + + let rows: [(&str, VerifiedAttestation, &str, Vec<String>); 7] = [ + ( + "tdx-up-to-date", + verified_attestation( + empty_tdx_quote(), + DstackVerifiedReport::DstackTdx(tdx_report("UpToDate", &[])), + ), + "UpToDate", + Vec::new(), + ), + ( + "tdx-out-of-date-carries-advisories", + verified_attestation( + empty_tdx_quote(), + DstackVerifiedReport::DstackTdx(tdx_report("OutOfDate", &advisories)), + ), + "OutOfDate", + expected_advisories.clone(), + ), + ( + "tdx-revoked", + verified_attestation( + empty_tdx_quote(), + DstackVerifiedReport::DstackTdx(tdx_report("Revoked", &advisories)), + ), + "Revoked", + expected_advisories.clone(), + ), + ( + // The bundled TPM report has no TCB surface: the TDX report must + // still drive policy, and must not be flattened to empty. + "gcp-tdx-reads-the-tdx-report-not-the-tpm-report", + verified_attestation( + AttestationQuote::DstackGcpTdx(DstackGcpTdxQuote { + tdx_quote: TdxQuote { + quote: Vec::new(), + event_log: Vec::new(), + }, + tpm_quote: TpmQuote { + message: Vec::new(), + signature: Vec::new(), + pcr_values: Vec::new(), + ak_cert: Vec::new(), + platform: dstack_types::Platform::Gcp, + event_log: Vec::new(), + }, + }), + DstackVerifiedReport::DstackGcpTdx { + tdx_report: tdx_report("OutOfDate", &advisories), + tpm_report: tpm_report_for_gcp(), + }, + ), + "OutOfDate", + expected_advisories.clone(), + ), + ( + "sev-snp-matching-tcb-versions-are-up-to-date", + verified_attestation( + AttestationQuote::DstackAmdSevSnp(SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + mr_config: String::new(), + }), + DstackVerifiedReport::DstackAmdSevSnp(snp_report(snp_up_to_date, &[])), + ), + "UpToDate", + Vec::new(), + ), + ( + "sev-snp-mismatched-tcb-versions-are-out-of-date", + verified_attestation( + AttestationQuote::DstackAmdSevSnp(SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + mr_config: String::new(), + }), + DstackVerifiedReport::DstackAmdSevSnp(snp_report(snp_out_of_date, &advisories)), + ), + "OutOfDate", + expected_advisories.clone(), + ), + ( + // No TCB surface, but normalized to UpToDate so the verifier's + // policy boot info matches the KMS bootAuth payload. + "aws-nitro-tpm-is-normalized-to-up-to-date", + verified_attestation( + AttestationQuote::DstackAwsNitroTpm(DstackAwsNitroTpmQuote { + attestation_doc: Vec::new(), + }), + DstackVerifiedReport::DstackAwsNitroTpm(AwsNitroTpmVerifiedReport { + module_id: String::new(), + pcrs: BTreeMap::new(), + public_key: None, + user_data: Vec::new(), + nonce: None, + timestamp: 0, + }), + ), + "UpToDate", + Vec::new(), + ), + ]; + + for (name, attestation, expected_status, expected_advisory_ids) in rows { + let (status, advisory_ids) = policy_tcb_fields(&attestation); + assert_eq!(status, expected_status, "{name}"); + assert_eq!(advisory_ids, expected_advisory_ids, "{name}"); + } + } + + /// Split out from the table above because it asserts the opposite property: + /// Nitro Enclave must stay empty so a relying party's "UpToDate" gate fails + /// closed rather than accepting a platform with no TCB surface. + #[test] + fn nitro_enclave_reports_no_tcb_status_and_fails_an_up_to_date_gate() { + let attestation = verified_attestation( + AttestationQuote::DstackNitroEnclave(DstackNitroQuote { + nsm_quote: Vec::new(), + }), + DstackVerifiedReport::DstackNitroEnclave(NitroVerifiedReport { + module_id: String::new(), + pcrs: NitroPcrs { + pcr0: vec![0x10; 48], + pcr1: vec![0x11; 48], + pcr2: vec![0x12; 48], + }, + user_data: Vec::new(), + timestamp: 0, + }), + ); + + let (status, advisory_ids) = policy_tcb_fields(&attestation); + assert_eq!(status, ""); + assert_ne!(status, "UpToDate"); + assert!(advisory_ids.is_empty()); + } + + #[test] + fn gcp_and_nitro_enclave_measurement_bindings_matrix() { + let verifier = test_verifier(); + + let nitro_pcrs = NitroPcrs { + pcr0: vec![0x10; 48], + pcr1: vec![0x11; 48], + pcr2: vec![0x12; 48], + }; + let nitro_config: VmConfig = serde_json::from_value(serde_json::json!({ + "os_image_hash": hex::encode(nitro_pcrs.image_hash()), + })) + .unwrap(); + verifier + .verify_os_image_hash_for_nitro_enclave(&nitro_config, &nitro_pcrs) + .unwrap(); + let mut changed_nitro = nitro_pcrs.clone(); + changed_nitro.pcr2[0] ^= 1; + assert!(verifier + .verify_os_image_hash_for_nitro_enclave(&nitro_config, &changed_nitro) + .is_err()); + let debug_nitro = NitroPcrs { + pcr0: vec![0; 48], + pcr1: vec![0; 48], + pcr2: vec![0; 48], + }; + assert!(verifier + .verify_os_image_hash_for_nitro_enclave(&nitro_config, &debug_nitro) + .is_err()); + + let uki_hash = vec![0x24; 32]; + let measurement = dstack_types::GcpOsImageMeasurement::new(uki_hash.clone()).unwrap(); + let measurement_bytes = measurement.to_cbor_vec(); + let checksum_file = format!( + "{} measurement.gcp.cbor\n", + hex::encode(Sha256::digest(&measurement_bytes)) + ) + .into_bytes(); + let os_image_hash = dstack_types::image_hash_from_sha256sum(&checksum_file); + let gcp_config: VmConfig = serde_json::from_value(serde_json::json!({ + "os_image_hash": hex::encode(os_image_hash), + "gcp_measurement": dstack_types::GcpOsImageMeasurementDocument::new( + checksum_file, + measurement_bytes, + ), + })) + .unwrap(); + let expected_pcr0 = + hex!("0cca9ec161b09288802e5a112255d21340ed5b797f5fe29cecccfd8f67b9f802"); + let gcp_quote = |pcr0: Vec<u8>, event_28: Vec<u8>| TpmQuote { + message: Vec::new(), + signature: Vec::new(), + pcr_values: vec![tpm_types::PcrValue { + index: 0, + algorithm: "sha256".into(), + value: pcr0, + }], + ak_cert: Vec::new(), + platform: dstack_types::Platform::Gcp, + event_log: vec![ + tpm_types::TpmEvent { + pcr_index: 2, + digest: vec![1; 32], + }, + tpm_types::TpmEvent { + pcr_index: 2, + digest: vec![2; 32], + }, + tpm_types::TpmEvent { + pcr_index: 2, + digest: event_28, + }, + ], + }; + verifier + .verify_os_image_hash_for_gcp_tdx( + &gcp_config, + &gcp_quote(expected_pcr0.to_vec(), uki_hash.clone()), + ) + .unwrap(); + assert!(verifier + .verify_os_image_hash_for_gcp_tdx( + &gcp_config, + &gcp_quote(vec![0; 32], uki_hash.clone()), + ) + .is_err()); + assert!(verifier + .verify_os_image_hash_for_gcp_tdx( + &gcp_config, + &gcp_quote(expected_pcr0.to_vec(), vec![0; 32]), + ) + .is_err()); + let mut missing_document = gcp_config; + missing_document.gcp_measurement = None; + assert!(verifier + .verify_os_image_hash_for_gcp_tdx( + &missing_document, + &gcp_quote(expected_pcr0.to_vec(), uki_hash), + ) + .is_err()); } #[test] @@ -1343,6 +1858,8 @@ mod tests { digest: vec![digest_byte; 48], event: name.to_string(), event_payload: TDX_ACPI_DATA_EVENT_PAYLOAD.to_vec(), + version: Default::default(), + preimage: None, } } @@ -1385,6 +1902,281 @@ mod tests { assert!(decode_key_provider_info(b"not json").is_none()); } + fn sample_measurements(byte: u8) -> TdxMeasurements { + TdxMeasurements { + mrtd: vec![byte; 48], + rtmr0: vec![byte.wrapping_add(1); 48], + rtmr1: vec![byte.wrapping_add(2); 48], + rtmr2: vec![byte.wrapping_add(3); 48], + } + } + + #[test] + fn measurement_cache_version_mismatch_is_ignored_and_replaced() { + let directory = tempfile::tempdir().unwrap(); + let mut verifier = test_verifier(); + verifier.image_cache_dir = directory.path().display().to_string(); + let config: VmConfig = serde_json::from_str("{}").unwrap(); + let key = CvmVerifier::vm_config_cache_key(&config).unwrap(); + let path = verifier.measurement_cache_path(&key); + fs_err::create_dir_all(path.parent().unwrap()).unwrap(); + + fs_err::write( + &path, + serde_json::to_vec(&CachedMeasurement { + version: MEASUREMENT_CACHE_VERSION - 1, + measurements: sample_measurements(0x11), + }) + .unwrap(), + ) + .unwrap(); + assert!(verifier + .load_measurements_from_cache(&key) + .unwrap() + .is_none()); + + let current = sample_measurements(0x22); + verifier + .store_measurements_in_cache(&key, ¤t) + .unwrap(); + let loaded = verifier + .load_measurements_from_cache(&key) + .unwrap() + .expect("current cache entry"); + assert_eq!( + serde_json::to_vec(&loaded).unwrap(), + serde_json::to_vec(¤t).unwrap() + ); + } + + #[test] + fn corrupt_measurement_cache_entry_is_ignored() { + let directory = tempfile::tempdir().unwrap(); + let mut verifier = test_verifier(); + verifier.image_cache_dir = directory.path().display().to_string(); + let config: VmConfig = serde_json::from_str("{}").unwrap(); + let key = CvmVerifier::vm_config_cache_key(&config).unwrap(); + let path = verifier.measurement_cache_path(&key); + fs_err::create_dir_all(path.parent().unwrap()).unwrap(); + fs_err::write(path, b"{not json").unwrap(); + + assert!(verifier + .load_measurements_from_cache(&key) + .unwrap() + .is_none()); + } + + #[test] + fn concurrent_measurement_cache_writes_are_atomic() { + let directory = tempfile::tempdir().unwrap(); + let mut verifier = test_verifier(); + verifier.image_cache_dir = directory.path().display().to_string(); + let config: VmConfig = serde_json::from_str("{}").unwrap(); + let key = CvmVerifier::vm_config_cache_key(&config).unwrap(); + let first = sample_measurements(0x11); + let second = sample_measurements(0x22); + + std::thread::scope(|scope| { + for index in 0..16 { + let verifier = &verifier; + let key = &key; + let measurements = if index % 2 == 0 { &first } else { &second }; + scope.spawn(move || { + verifier + .store_measurements_in_cache(key, measurements) + .unwrap(); + }); + } + }); + let cached = verifier + .load_measurements_from_cache(&key) + .unwrap() + .expect("one complete cache entry"); + let encoded = serde_json::to_vec(&cached).unwrap(); + assert!( + encoded == serde_json::to_vec(&first).unwrap() + || encoded == serde_json::to_vec(&second).unwrap() + ); + let entries = fs_err::read_dir(verifier.measurement_cache_dir()) + .unwrap() + .collect::<Result<Vec<_>, _>>() + .unwrap(); + assert_eq!(entries.len(), 1, "temporary cache files must not survive"); + } + + #[test] + fn image_cache_pruning_keeps_checksum_identity() { + let dir = tempfile::tempdir().expect("temp image directory"); + let files_doc = "00 metadata.json\n"; + fs_err::write(dir.path().join("sha256sum.txt"), files_doc).unwrap(); + fs_err::write(dir.path().join("metadata.json"), "{}").unwrap(); + fs_err::write(dir.path().join("unmeasured"), "remove me").unwrap(); + + CvmVerifier::prune_unlisted_image_files(dir.path(), files_doc).unwrap(); + + assert!(dir.path().join("sha256sum.txt").exists()); + assert!(dir.path().join("metadata.json").exists()); + assert!(!dir.path().join("unmeasured").exists()); + } + + #[test] + fn image_paths_must_be_confined_and_manifest_paths_must_be_flat() { + for path in ["../escape", "/absolute", "nested/../escape"] { + assert!( + !CvmVerifier::is_confined_archive_path(Path::new(path)), + "{path}" + ); + } + // `.` components are stripped by `unpack_in` and cannot escape, so the + // check must accept them: `tar -czf out.tar.gz .` prefixes every member + // with `./` and published images are packed that way. + for path in ["nested/artifact", "./metadata.json", ".", "./", ""] { + assert!( + CvmVerifier::is_confined_archive_path(Path::new(path)), + "{path}" + ); + } + + let digest = "00".repeat(32); + assert!( + CvmVerifier::validate_image_manifest_paths(&format!("{digest} metadata.json\n")) + .is_ok() + ); + for path in [ + "../escape", + "/absolute", + "nested/artifact", + "./metadata.json", + ".", + "sha256sum.txt", + ] { + assert!( + CvmVerifier::validate_image_manifest_paths(&format!("{digest} {path}\n")).is_err(), + "{path}" + ); + } + } + + #[test] + fn image_archive_rejects_links_and_accepts_regular_files() { + let directory = tempfile::tempdir().unwrap(); + let valid = directory.path().join("valid.tar.gz"); + { + let file = fs_err::File::create(&valid).unwrap(); + let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::default()); + let mut archive = tar::Builder::new(encoder); + let payload = b"artifact"; + let mut header = tar::Header::new_gnu(); + header.set_size(payload.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, "nested/artifact", &payload[..]) + .unwrap(); + archive.finish().unwrap(); + } + let output = directory.path().join("valid-output"); + fs_err::create_dir(&output).unwrap(); + CvmVerifier::extract_image_archive(&valid, &output).unwrap(); + assert_eq!( + fs_err::read(output.join("nested/artifact")).unwrap(), + b"artifact" + ); + + let linked = directory.path().join("linked.tar.gz"); + { + let file = fs_err::File::create(&linked).unwrap(); + let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::default()); + let mut archive = tar::Builder::new(encoder); + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Symlink); + header.set_size(0); + header.set_mode(0o777); + header.set_link_name("../outside").unwrap(); + header.set_cksum(); + archive.append_data(&mut header, "link", &[][..]).unwrap(); + archive.finish().unwrap(); + } + let output = directory.path().join("linked-output"); + fs_err::create_dir(&output).unwrap(); + assert!(CvmVerifier::extract_image_archive(&linked, &output).is_err()); + } + + /// Images published on download.dstack.org come in two shapes: members + /// packed from a glob (`bzImage`, ...) and members packed from `.` + /// (`./`, `./bzImage`, ...). Both must extract to the same flat layout. + #[test] + fn image_archive_accepts_dot_prefixed_members() { + let directory = tempfile::tempdir().unwrap(); + let archive_path = directory.path().join("dot-prefixed.tar.gz"); + { + let file = fs_err::File::create(&archive_path).unwrap(); + let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::default()); + let mut archive = tar::Builder::new(encoder); + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Directory); + header.set_size(0); + header.set_mode(0o755); + header.set_cksum(); + archive.append_data(&mut header, "./", &[][..]).unwrap(); + let payload = b"artifact"; + let mut header = tar::Header::new_gnu(); + header.set_size(payload.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, "./metadata.json", &payload[..]) + .unwrap(); + archive.finish().unwrap(); + } + let output = directory.path().join("dot-prefixed-output"); + fs_err::create_dir(&output).unwrap(); + CvmVerifier::extract_image_archive(&archive_path, &output).unwrap(); + assert_eq!( + fs_err::read(output.join("metadata.json")).unwrap(), + b"artifact" + ); + } + + /// `GzDecoder` stops at the first member of a concatenated gzip stream and + /// reports clean EOF, which would truncate the archive without an error. + #[test] + fn image_archive_reads_every_gzip_member() { + let directory = tempfile::tempdir().unwrap(); + let mut tarball = tar::Builder::new(Vec::new()); + let payload = b"artifact"; + let mut header = tar::Header::new_gnu(); + header.set_size(payload.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + tarball + .append_data(&mut header, "metadata.json", &payload[..]) + .unwrap(); + let tarball = tarball.into_inner().unwrap(); + + let archive_path = directory.path().join("multi-member.tar.gz"); + { + use std::io::Write; + + let mut file = fs_err::File::create(&archive_path).unwrap(); + // One gzip member per half of the tar stream. + for half in tarball.chunks(tarball.len().div_ceil(2)) { + let mut encoder = + flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(half).unwrap(); + file.write_all(&encoder.finish().unwrap()).unwrap(); + } + file.flush().unwrap(); + } + let output = directory.path().join("multi-member-output"); + fs_err::create_dir(&output).unwrap(); + CvmVerifier::extract_image_archive(&archive_path, &output).unwrap(); + assert_eq!( + fs_err::read(output.join("metadata.json")).unwrap(), + b"artifact" + ); + } + #[tokio::test] async fn verifies_sev_snp_attestation_fixture_without_image_download() { let request: VerificationRequest = @@ -1396,7 +2188,7 @@ mod tests { image_cache_dir.display().to_string(), "http://127.0.0.1:9/should-not-download/{OS_IMAGE_HASH}.tar.gz".to_string(), Duration::from_secs(1), - None, + test_attestation_verifier(), ); let response = verifier.verify(request).await.expect("verifier runs"); @@ -1406,8 +2198,8 @@ mod tests { assert!(response.details.os_image_hash_verified); assert!(!response.details.acpi_tables_verified); assert_eq!( - response.details.attestation_mode, - Some(ra_tls::attestation::AttestationMode::DstackAmdSevSnp) + response.details.tee_variant, + Some(ra_tls::attestation::TeeVariant::DstackAmdSevSnp) ); assert!( !image_cache_dir.exists(), @@ -1428,19 +2220,22 @@ mod tests { cache.path().join("cache").display().to_string(), "http://127.0.0.1:9/should-not-download/{OS_IMAGE_HASH}.tar.gz".to_string(), Duration::from_secs(1), - None, + test_attestation_verifier(), ); let response = verifier.verify(request).await.expect("verifier runs"); assert!(response.is_valid, "{:?}", response.reason); assert_eq!( - response.details.attestation_mode, - Some(ra_tls::attestation::AttestationMode::DstackAmdSevSnp) + response.details.tee_variant, + Some(ra_tls::attestation::TeeVariant::DstackAmdSevSnp) ); } + /// The fixture was captured from a real CVM, so its RTMR0 ACPI digests are + /// whatever QEMU actually produced. Reproducing them without the image + /// proves the generator agrees with hardware, not just with itself. #[tokio::test] - async fn verifies_tdx_lite_fixture_without_acpi_table_verification() { + async fn verifies_tdx_lite_fixture_without_image_download() { let request: VerificationRequest = serde_json::from_str(include_str!("../fixtures/tdx-lite-attestation.json")) .expect("TDX lite verifier fixture parses"); @@ -1450,7 +2245,7 @@ mod tests { image_cache_dir.display().to_string(), "http://127.0.0.1:9/should-not-download/{OS_IMAGE_HASH}.tar.gz".to_string(), Duration::from_secs(1), - None, + test_attestation_verifier(), ); let response = verifier.verify(request).await.expect("verifier runs"); @@ -1458,14 +2253,461 @@ mod tests { assert!(response.details.quote_verified); assert!(response.details.event_log_verified); assert!(response.details.os_image_hash_verified); - assert!(!response.details.acpi_tables_verified); + assert!(response.details.acpi_tables_verified); + assert_eq!( + response.details.tee_variant, + Some(ra_tls::attestation::TeeVariant::DstackTdx) + ); + assert!( + !image_cache_dir.exists(), + "TDX lite verification must not download or cache OS images" + ); + } + + /// The guest agent's v1 surfaces always hand out the MessagePack V1 schema, + /// while every captured fixture is the legacy SCALE form. Re-encoding a + /// fixture as V1 must not change a single verification outcome: the same + /// quote, event log, image and ACPI checks run, and they reach the same + /// verdict with the same details. + #[tokio::test] + async fn msgpack_and_scale_encodings_verify_identically() { + let fixtures = [ + ( + "tdx-lite", + include_str!("../fixtures/tdx-lite-attestation.json"), + ), + ( + "tdx-lite-normalized", + include_str!("../fixtures/tdx-lite-normalized-attestation.json"), + ), + ( + "tdx-lite-normalized-qemu-10-2", + include_str!("../fixtures/tdx-lite-normalized-qemu-10-2-attestation.json"), + ), + ( + "sev-snp", + include_str!("../fixtures/sev-snp-attestation.json"), + ), + ]; + + for (name, fixture) in fixtures { + let request: VerificationRequest = + serde_json::from_str(fixture).expect("verifier fixture parses"); + let scale = request + .attestation + .clone() + .expect("fixture carries an attestation"); + let legacy = VersionedAttestation::from_bytes(&scale).expect("fixture decodes"); + assert!( + matches!(legacy, VersionedAttestation::V0 { .. }), + "{name}: fixture is expected to be the legacy form" + ); + let msgpack = VersionedAttestation::V1 { + attestation: legacy.into_v1(), + } + .to_bytes() + .expect("attestation re-encodes as V1"); + assert!( + matches!(msgpack.first(), Some(0x80..=0x8f | 0xde | 0xdf)), + "{name}: re-encoded attestation is not a MessagePack map" + ); + + let mut responses = Vec::new(); + for attestation in [scale, msgpack] { + let cache = tempfile::tempdir().expect("temp cache dir"); + let verifier = CvmVerifier::new( + cache.path().join("cache").display().to_string(), + "http://127.0.0.1:9/should-not-download/{OS_IMAGE_HASH}.tar.gz".to_string(), + Duration::from_secs(1), + test_attestation_verifier(), + ); + let response = verifier + .verify(VerificationRequest { + attestation: Some(attestation), + ..request.clone() + }) + .await + .expect("verifier runs"); + assert!(response.is_valid, "{name}: {:?}", response.reason); + assert!(response.details.quote_verified, "{name}"); + assert!(response.details.event_log_verified, "{name}"); + assert!(response.details.os_image_hash_verified, "{name}"); + responses.push(serde_json::to_value(&response).expect("response serializes")); + } + assert_eq!( + responses[0], responses[1], + "{name}: MessagePack and SCALE encodings verified differently" + ); + } + } + + /// Verify one attestation through the full `CvmVerifier` path with image + /// download disabled, and require every check to pass. + async fn verify_lite_attestation(name: &str, attestation: Vec<u8>) -> serde_json::Value { + let cache = tempfile::tempdir().expect("temp cache dir"); + let verifier = CvmVerifier::new( + cache.path().join("cache").display().to_string(), + "http://127.0.0.1:9/should-not-download/{OS_IMAGE_HASH}.tar.gz".to_string(), + Duration::from_secs(1), + test_attestation_verifier(), + ); + let response = verifier + .verify(VerificationRequest { + quote: None, + event_log: None, + vm_config: None, + attestation: Some(attestation), + debug: None, + }) + .await + .expect("verifier runs"); + assert!(response.is_valid, "{name}: {:?}", response.reason); + assert!(response.details.quote_verified, "{name}"); + assert!(response.details.event_log_verified, "{name}"); + assert!(response.details.os_image_hash_verified, "{name}"); + assert!(response.details.acpi_tables_verified, "{name}"); + serde_json::to_value(&response).expect("response serializes") + } + + fn fixture_attestation(fixture: &str) -> Vec<u8> { + let request: VerificationRequest = + serde_json::from_str(fixture).expect("verifier fixture parses"); + request.attestation.expect("fixture carries an attestation") + } + + fn is_msgpack_v1(bytes: &[u8]) -> bool { + matches!( + VersionedAttestation::from_bytes(bytes), + Ok(VersionedAttestation::V1 { .. }) + ) && matches!(bytes.first(), Some(0x80..=0x8f | 0xde | 0xdf)) + } + + /// Captured from a real TDX CVM: `/v1/Attest` and the frozen `/Attest` + /// asked for the same report data in the same boot. The platform produced + /// the legacy form (V1 runtime events), so the v1 bytes are the guest + /// agent's own MessagePack re-encoding, and they must verify to exactly the + /// response the legacy bytes do. + #[tokio::test] + async fn verifies_real_v1_attest_identically_to_v0_from_the_same_boot() { + let v1 = fixture_attestation(include_str!("../fixtures/tdx-lite-v1-attest.json")); + let v0 = fixture_attestation(include_str!("../fixtures/tdx-lite-v0-attest.json")); + assert!(is_msgpack_v1(&v1), "/v1/Attest must be MessagePack V1"); + assert_eq!(v0.first(), Some(&0x00), "/Attest must be legacy SCALE"); + + let v1_response = verify_lite_attestation("v1 Attest", v1).await; + let v0_response = verify_lite_attestation("v0 Attest", v0).await; + assert_eq!( + v1_response, v0_response, + "v1 and v0 Attest from one boot verified differently" + ); + } + + /// Captured from the same boot: the certificate chains `/v1/IssueCert` + /// and the frozen `/GetTlsKey` returned with `usage_ra_tls` set, signed by + /// the CVM's local CA. The v1 leaf embeds MessagePack V1. Each leaf must + /// chain to its CA, pass RA-TLS verification with the report data bound to + /// its own public key, and carry an attestation that passes the full image + /// check -- and both must name the same CVM. + #[tokio::test] + async fn verifies_real_v1_issue_cert_chain_and_embedded_attestation() { + use ra_tls::traits::CertExt as _; + + let mut identities = Vec::new(); + for (name, pem, expect_v1) in [ + ( + "v1 IssueCert", + include_str!("../fixtures/tdx-lite-v1-issue-cert.pem"), + true, + ), + ( + "v0 GetTlsKey", + include_str!("../fixtures/tdx-lite-v0-get-tls-key.pem"), + false, + ), + ] { + let chain: Vec<_> = x509_parser::pem::Pem::iter_from_buffer(pem.as_bytes()) + .map(|pem| pem.expect("PEM block parses")) + .collect(); + assert_eq!(chain.len(), 2, "{name}: leaf and CA"); + let leaf = chain[0].parse_x509().expect("leaf parses"); + let ca = chain[1].parse_x509().expect("CA parses"); + leaf.verify_signature(Some(ca.public_key())) + .unwrap_or_else(|err| panic!("{name}: leaf is not signed by its CA: {err}")); + + let embedded = leaf + .get_extension_bytes(ra_tls::oids::PHALA_RATLS_ATTESTATION) + .expect("extension reads") + .expect("leaf carries an attestation"); + assert_eq!(is_msgpack_v1(&embedded), expect_v1, "{name}: wire form"); + + let verified = + ra_tls::attestation::verify_der(&chain[0].contents, &test_attestation_verifier()) + .await + .unwrap_or_else(|err| panic!("{name}: RA-TLS verification failed: {err:#}")); + assert_eq!( + verified.public_key_der, + leaf.public_key().raw, + "{name}: attestation must be bound to the leaf key" + ); + + let response = verify_lite_attestation(name, embedded).await; + identities.push(response["details"]["app_info"].clone()); + } assert_eq!( - response.details.attestation_mode, - Some(ra_tls::attestation::AttestationMode::DstackTdx) + identities[0], identities[1], + "v1 and v0 certificates from one boot name different CVMs" + ); + } + + /// The certificate verification must not be vacuous: a leaf whose embedded + /// attestation was captured for a different key is rejected. + #[tokio::test] + async fn rejects_a_real_v1_certificate_attestation_bound_to_another_key() { + let v1_leaf = x509_parser::pem::Pem::iter_from_buffer( + include_str!("../fixtures/tdx-lite-v1-issue-cert.pem").as_bytes(), + ) + .next() + .expect("leaf present") + .expect("PEM block parses"); + let v0_leaf = x509_parser::pem::Pem::iter_from_buffer( + include_str!("../fixtures/tdx-lite-v0-get-tls-key.pem").as_bytes(), + ) + .next() + .expect("leaf present") + .expect("PEM block parses"); + // The v1 attestation checked against the v0 leaf's key: same quote, same + // CVM, wrong binding. + let v1_cert = v1_leaf.parse_x509().expect("leaf parses"); + let v0_cert = v0_leaf.parse_x509().expect("leaf parses"); + let attestation = ra_tls::attestation::from_der(&v1_leaf.contents) + .expect("extension decodes") + .expect("leaf carries an attestation"); + assert!(is_msgpack_v1( + &attestation.clone().to_bytes().expect("re-encodes") + )); + assert_ne!(v1_cert.public_key().raw, v0_cert.public_key().raw); + let err = attestation + .into_v1() + .verify_with_ra_pubkey(v0_cert.public_key().raw, &test_attestation_verifier()) + .await + .err() + .expect("an attestation bound to another key must not verify"); + assert!( + err.to_string().contains("report data mismatch"), + "unexpected error: {err:#}" + ); + } + + /// Rebuild the fixture's measurement document around a different kernel + /// command line, keeping every hash that commits to it consistent: + /// `measurement.tdx.cbor`, its `sha256sum.txt` entry, and the + /// `os_image_hash` over that file. Nothing in the image-identity chain can + /// object to the result. + fn forge_command_line(attestation: &mut ra_tls::attestation::Attestation, suffix: &str) { + use sha2::{Digest, Sha256}; + + let mut config: serde_json::Value = + serde_json::from_str(&attestation.config).expect("vm_config parses"); + let mut document: dstack_types::TdxOsImageMeasurementDocument = + serde_json::from_value(config["tdx_measurement"].clone()) + .expect("tdx_measurement parses"); + + let mut measurement = + dstack_types::TdxOsImageMeasurement::from_cbor_slice(&document.measurement) + .expect("measurement decodes"); + let previous = Sha256::digest(&document.measurement); + measurement.image.base_cmdline.push_str(suffix); + document.measurement = measurement.to_cbor_vec(); + + let checksum = String::from_utf8(document.checksum_file).expect("checksum file is utf-8"); + let mut rewritten = String::new(); + for line in checksum.lines() { + let (hash, name) = line.split_once(" ").expect("checksum line"); + if name == "measurement.tdx.cbor" { + assert_eq!(hash, hex::encode(previous), "fixture checksum is stale"); + rewritten.push_str(&hex::encode(Sha256::digest(&document.measurement))); + } else { + rewritten.push_str(hash); + } + rewritten.push_str(" "); + rewritten.push_str(name); + rewritten.push('\n'); + } + document.checksum_file = rewritten.into_bytes(); + + config["os_image_hash"] = + serde_json::json!(hex::encode(Sha256::digest(&document.checksum_file))); + config["tdx_measurement"] = serde_json::to_value(&document).expect("document serializes"); + attestation.config = config.to_string(); + } + + /// The document carries the kernel command line itself, not a digest of it, + /// so nothing stops a host from writing a different one and rebuilding + /// every hash that commits to it. What stops it is RTMR[2], which the CVM + /// extended with the command line it was actually booted with. + /// + /// This forges exactly that: an image identity that is internally + /// consistent and therefore passes every `os_image_hash` check. Only the + /// measurement comparison can reject it. + #[tokio::test] + async fn tdx_lite_rejects_a_self_consistent_forged_command_line() { + let request: VerificationRequest = + serde_json::from_str(include_str!("../fixtures/tdx-lite-attestation.json")) + .expect("TDX lite verifier fixture parses"); + let bytes = request.attestation.expect("fixture carries an attestation"); + + let mut attestation = + match VersionedAttestation::from_bytes(&bytes).expect("attestation decodes") { + VersionedAttestation::V0 { attestation } => attestation, + VersionedAttestation::V1 { .. } => { + panic!("fixture is expected to be a legacy attestation") + } + }; + forge_command_line(&mut attestation, " forged=1"); + let forged = VersionedAttestation::V0 { attestation } + .to_bytes() + .expect("attestation re-encodes"); + + let cache = tempfile::tempdir().expect("temp cache dir"); + let verifier = CvmVerifier::new( + cache.path().join("cache").display().to_string(), + "http://127.0.0.1:9/should-not-download/{OS_IMAGE_HASH}.tar.gz".to_string(), + Duration::from_secs(1), + test_attestation_verifier(), + ); + + let response = verifier + .verify(VerificationRequest { + quote: None, + event_log: None, + vm_config: None, + attestation: Some(forged), + debug: None, + }) + .await + .expect("verifier runs"); + + assert!(!response.is_valid, "a forged command line must not verify"); + assert!(!response.details.os_image_hash_verified); + let reason = response.reason.unwrap_or_default(); + assert!( + reason.contains("RTMR2 mismatch"), + "expected the command line to be caught by RTMR[2], got: {reason}" ); + } + + /// Captured from a CVM whose OVMF normalizes the Linux setup header, so + /// RTMR[1] is the plain Authenticode hash of the shipped kernel. The host + /// ran QEMU 8.2.2 -- a version that *does* rewrite the header -- so this + /// only passes if the firmware actually undid that rewrite. + #[tokio::test] + async fn verifies_tdx_lite_fixture_with_normalized_kernel_header() { + let request: VerificationRequest = serde_json::from_str(include_str!( + "../fixtures/tdx-lite-normalized-attestation.json" + )) + .expect("normalized TDX lite verifier fixture parses"); + let cache = tempfile::tempdir().expect("temp cache dir"); + let image_cache_dir = cache.path().join("cache"); + let verifier = CvmVerifier::new( + image_cache_dir.display().to_string(), + "http://127.0.0.1:9/should-not-download/{OS_IMAGE_HASH}.tar.gz".to_string(), + Duration::from_secs(1), + test_attestation_verifier(), + ); + + let response = verifier.verify(request).await.expect("verifier runs"); + assert!(response.is_valid, "{:?}", response.reason); + assert!(response.details.quote_verified); + assert!(response.details.event_log_verified); + assert!(response.details.os_image_hash_verified); + assert!(response.details.acpi_tables_verified); assert!( !image_cache_dir.exists(), "TDX lite verification must not download or cache OS images" ); } + + /// The same image as the fixture above, captured on QEMU 10.2.1 -- a + /// version that does *not* rewrite the setup header. Its RTMR[1] is + /// byte-for-byte the one the 8.2.2 capture produced, which is the whole + /// point of normalizing: the digest no longer depends on the host's QEMU. + /// MRTD and RTMR[0] do differ, because page-add ordering and the generated + /// ACPI tables genuinely are version-specific. + #[tokio::test] + async fn verifies_tdx_lite_fixture_with_normalized_kernel_header_on_qemu_10_2() { + let request: VerificationRequest = serde_json::from_str(include_str!( + "../fixtures/tdx-lite-normalized-qemu-10-2-attestation.json" + )) + .expect("QEMU 10.2 normalized TDX lite verifier fixture parses"); + let cache = tempfile::tempdir().expect("temp cache dir"); + let image_cache_dir = cache.path().join("cache"); + let verifier = CvmVerifier::new( + image_cache_dir.display().to_string(), + "http://127.0.0.1:9/should-not-download/{OS_IMAGE_HASH}.tar.gz".to_string(), + Duration::from_secs(1), + test_attestation_verifier(), + ); + + let response = verifier.verify(request).await.expect("verifier runs"); + assert!(response.is_valid, "{:?}", response.reason); + assert!(response.details.quote_verified); + assert!(response.details.event_log_verified); + assert!(response.details.os_image_hash_verified); + assert!(response.details.acpi_tables_verified); + assert!( + !image_cache_dir.exists(), + "TDX lite verification must not download or cache OS images" + ); + } + + /// The captured VM ran 2 vCPUs; a VM shape that disagrees with the quote + /// must not reproduce its ACPI digests, which is what makes the recomputed + /// digests worth comparing in the first place. + #[test] + fn tdx_lite_acpi_hashes_depend_on_the_reported_vm_shape() { + let fixture: serde_json::Value = + serde_json::from_str(include_str!("../fixtures/tdx-lite-getquote.json")) + .expect("TDX lite getquote fixture parses"); + let mut vm_config: VmConfig = serde_json::from_str( + fixture["vm_config"] + .as_str() + .expect("vm_config is a string"), + ) + .expect("fixture vm_config parses"); + let event_log: Vec<TdxEvent> = serde_json::from_str( + fixture["event_log"] + .as_str() + .expect("event_log is a string"), + ) + .expect("fixture event log parses"); + let ovmf_variant = vm_config.ovmf_variant.unwrap_or_default(); + + let reported = CvmVerifier::tdx_acpi_hashes_from_event_log(&event_log) + .expect("fixture carries named ACPI digests"); + let expected = dstack_mr::tdx::expected_rtmr0_acpi_hashes(&vm_config, ovmf_variant) + .expect("ACPI tables are generated for the fixture VM shape"); + CvmVerifier::assert_tdx_acpi_hashes_match(&expected, &reported) + .expect("recomputed digests match the captured CVM"); + + vm_config.cpu_count += 1; + let expected = dstack_mr::tdx::expected_rtmr0_acpi_hashes(&vm_config, ovmf_variant) + .expect("ACPI tables are generated for the altered VM shape"); + CvmVerifier::assert_tdx_acpi_hashes_match(&expected, &reported) + .expect_err("an extra vCPU must change the ACPI tables"); + } + + #[test] + fn tdx_lite_acpi_hash_mismatch_names_the_table() { + let expected = TdxRtmr0AcpiHashes { + loader: vec![1; 48], + rsdp: vec![2; 48], + tables: vec![3; 48], + }; + let mut reported = expected.clone(); + reported.tables = vec![4; 48]; + let err = CvmVerifier::assert_tdx_acpi_hashes_match(&expected, &reported) + .expect_err("mismatched tables digest is rejected"); + assert!(err.to_string().contains(TDX_ACPI_TABLES_EVENT), "{err:#}"); + assert!(CvmVerifier::assert_tdx_acpi_hashes_match(&expected, &expected).is_ok()); + } } diff --git a/dstack/vmm/Cargo.toml b/dstack/vmm/Cargo.toml index 9dbc6399c..22387fcef 100644 --- a/dstack/vmm/Cargo.toml +++ b/dstack/vmm/Cargo.toml @@ -15,6 +15,7 @@ rocket-vsock-listener = { workspace = true } tracing.workspace = true tracing-subscriber = { workspace = true, features = ["env-filter"] } anyhow.workspace = true +libc.workspace = true serde = { workspace = true, features = ["derive"] } serde_json.workspace = true shared_child.workspace = true @@ -24,7 +25,7 @@ sha2.workspace = true hex.workspace = true fs-err.workspace = true getrandom = { workspace = true, features = ["std"] } -nix = { workspace = true, features = ["user"] } +nix = { workspace = true, features = ["fs", "process", "signal", "user"] } dirs.workspace = true which.workspace = true clap = { workspace = true, features = ["derive", "string"] } @@ -32,13 +33,15 @@ humantime.workspace = true strip-ansi-escapes.workspace = true tailf.workspace = true tokio = { workspace = true, features = ["full"] } -git-version.workspace = true -rocket-apitoken.workspace = true +dstack-api-auth.workspace = true +dstack-build-info.workspace = true serde_ini.workspace = true supervisor-client.workspace = true ra-rpc = { workspace = true, features = ["client", "rocket", "openapi"] } dstack-vmm-rpc.workspace = true +dstack-netd-rpc.workspace = true +http-client = { workspace = true, features = ["prpc"] } dstack-kms-rpc.workspace = true path-absolutize.workspace = true host-api.workspace = true @@ -46,9 +49,9 @@ safe-write.workspace = true guest-api = { workspace = true, features = ["client"] } load_config.workspace = true key-provider-client.workspace = true -dstack-port-forward.workspace = true dstack-types.workspace = true dstack-mr.workspace = true +mock-attestation = { path = "../crates/mock-attestation" } hex_fmt.workspace = true lspci.workspace = true base64.workspace = true @@ -59,8 +62,12 @@ fscommon.workspace = true or-panic.workspace = true url.workspace = true reqwest.workspace = true +rand.workspace = true flate2.workspace = true tar.workspace = true +tempfile.workspace = true +wait-timeout.workspace = true +listenfd.workspace = true [dev-dependencies] insta.workspace = true diff --git a/dstack/vmm/netd-rpc/Cargo.toml b/dstack/vmm/netd-rpc/Cargo.toml new file mode 100644 index 000000000..e3608acb0 --- /dev/null +++ b/dstack/vmm/netd-rpc/Cargo.toml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "dstack-netd-rpc" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +prpc.workspace = true +prost.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true + +[build-dependencies] +prpc-build.workspace = true diff --git a/dstack/vmm/netd-rpc/build.rs b/dstack/vmm/netd-rpc/build.rs new file mode 100644 index 000000000..2e6abea08 --- /dev/null +++ b/dstack/vmm/netd-rpc/build.rs @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +#![allow(clippy::expect_used)] + +fn main() { + prpc_build::configure() + .out_dir(std::env::var_os("OUT_DIR").expect("OUT_DIR not set")) + .mod_prefix("super::") + .build_scale_ext(false) + .disable_service_name_emission() + .disable_package_emission() + .enable_serde_extension() + .compile_dir("./proto") + .expect("failed to compile proto files"); +} diff --git a/dstack/vmm/netd-rpc/proto/netd_rpc.proto b/dstack/vmm/netd-rpc/proto/netd_rpc.proto new file mode 100644 index 000000000..058a83a93 --- /dev/null +++ b/dstack/vmm/netd-rpc/proto/netd_rpc.proto @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +import "google/protobuf/empty.proto"; + +package netd_rpc; + +// Names one host interface. The interface name is derived from these fields, +// so the same identity always refers to the same interface. +message InterfaceIdentity { + string instance_id = 1; + string vm_id = 2; + uint32 nic_index = 3; +} + +message PrepareBridgeRequest { + InterfaceIdentity identity = 1; + string bridge = 2; + string mac = 3; + uint32 qemu_uid = 4; + // Whether to bind an nwfilter to the TAP. Which filter, and with what + // parameters, comes from netd's own configuration. + bool filtered = 5; + // virtio-net queue pairs. Zero or one creates a single-queue TAP. + uint32 queues = 6; +} + +message PrepareMacvtapRequest { + InterfaceIdentity identity = 1; + string parent = 2; + string mac = 3; + uint32 qemu_uid = 4; + // macvtap mode; empty selects netd's default. + string mode = 5; + // virtio-net queue pairs. Zero or one creates a single-queue device. + uint32 queues = 6; +} + +// What netd built, so the caller can verify it before handing it to QEMU. +message PreparedInterface { + string tap = 1; + // Character device QEMU opens for a macvtap interface. Empty for a TAP. + string device = 2; + // Queue pairs the interface was created with. + uint32 queues = 3; +} + +message CheckInterfaceRequest { + InterfaceIdentity identity = 1; + // Also require the nwfilter binding. + bool filtered = 2; +} + +// Privileged broker for bridge TAPs, macvtap devices and libvirt nwfilter +// bindings. +service Netd { + // Create, or recreate, a bridge TAP and its optional nwfilter binding. + rpc PrepareBridge(PrepareBridgeRequest) returns (PreparedInterface); + // Create, or recreate, a macvtap device. + rpc PrepareMacvtap(PrepareMacvtapRequest) returns (PreparedInterface); + // Delete an interface and its binding. Deleting one that does not exist + // succeeds. + rpc RemoveInterface(InterfaceIdentity) returns (google.protobuf.Empty); + // Verify an interface, and optionally its binding, exists. For operations + // and integration diagnostics; launches use the Prepare calls. + rpc CheckInterface(CheckInterfaceRequest) returns (google.protobuf.Empty); +} diff --git a/dstack/vmm/netd-rpc/src/generated/mod.rs b/dstack/vmm/netd-rpc/src/generated/mod.rs new file mode 100644 index 000000000..a6ee06169 --- /dev/null +++ b/dstack/vmm/netd-rpc/src/generated/mod.rs @@ -0,0 +1,6 @@ +#![allow(async_fn_in_trait)] + +pub const FILE_DESCRIPTOR_SET: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/file_descriptor_set.bin")); + +include!(concat!(env!("OUT_DIR"), "/netd_rpc.rs")); diff --git a/dstack/vmm/netd-rpc/src/lib.rs b/dstack/vmm/netd-rpc/src/lib.rs new file mode 100644 index 000000000..aa5357d83 --- /dev/null +++ b/dstack/vmm/netd-rpc/src/lib.rs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +// +// SPDX-License-Identifier: Apache-2.0 + +//! RPC protocol between the VMM and `netd`, its privileged host networking +//! broker. + +extern crate alloc; + +pub use generated::*; + +mod generated; diff --git a/dstack/vmm/requirements.txt b/dstack/vmm/requirements.txt index 2028e3618..5a4636513 100644 --- a/dstack/vmm/requirements.txt +++ b/dstack/vmm/requirements.txt @@ -1,6 +1,6 @@ annotated-types==0.7.0 cffi==1.17.1 -cryptography==48.0.1 +cryptography==50.0.0 cytoolz==1.0.1 eth-hash==0.7.1 eth-keys==0.7.0 diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 3a4e53e0d..62bf381be 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -39,19 +39,34 @@ message VmInfo { repeated GuestEvent events = 14; // Effective network interfaces resolved against node config. repeated NetworkInterfaceStatus interfaces = 15; + // Whether a QEMU process exists for this VM right now. The interfaces above + // are what it built only while this is true; otherwise they are what the next + // launch would build. + bool running = 16; } // Runtime status of a resolved VM network interface. message NetworkInterfaceStatus { - // Product-facing mode: "user", "bridge", or "custom". + // Product-facing mode: "user", "bridge", "macvtap", or "custom". string mode = 1; - // QEMU/network backend shape: "slirp", "tap_bridge", or "custom". + // QEMU/network backend shape: "slirp", "tap_bridge", "macvtap", or "custom". string backend = 2; string mac = 3; // Linux bridge name for tap_bridge backend. optional string bridge_name = 4; // QEMU netdev id, e.g. "net0". optional string netdev_id = 5; + // Effective vhost-net data plane state for this interface. Absent for custom + // mode, where the operator supplies the whole netdev string and the VMM does + // not parse it, so it knows of no data-plane state to report. + optional bool vhost = 6; + // Effective virtio-net queue pairs. Absent for custom mode, for the same + // reason as vhost. + optional uint32 queues = 7; + // Effective macvtap forwarding mode, for macvtap interfaces only. Node + // configuration decides it, so it is reported here with the rest of the + // resolved state rather than on the VM's own NetworkingConfig. + optional string macvtap_mode = 8; } // Structured log or lifecycle event emitted by the guest or runtime. @@ -118,14 +133,30 @@ message VmConfiguration { optional NetworkingConfig networking = 18; // Per-VM networking overrides. If set, wins over singular networking. repeated NetworkingConfig networks = 19; + reserved 20; + // Development-only TEE ABI to simulate for this VM. When set, the VMM + // launches a non-TEE VM and provides an instance-specific simulator config. + optional string simulated_tee = 21; } // Per-VM networking configuration. message NetworkingConfig { - // Networking mode: "bridge", "user" + // Networking mode: "bridge", "user", "macvtap" string mode = 1; // Per-VM bridge interface name. Empty = node default bridge. string bridge_name = 2; + // Parent host interface for macvtap mode. + string parent = 3; + // Deployment requests must leave this empty: the forwarding mode is node + // configuration, and a VM never pins one. The effective value is reported on + // NetworkInterfaceStatus.macvtap_mode instead. + string macvtap_mode = 4; + // Move packet processing into the host kernel vhost-net data plane. Unset + // inherits the node default. User mode has no vhost backend and ignores it. + optional bool vhost = 5; + // virtio-net queue pairs. Unset inherits the node default. Bounded by the + // node's cvm.max_net_queues. + optional uint32 queues = 6; } // Requested GPU layout for a CVM. @@ -151,6 +182,16 @@ message PortMapping { uint32 vm_port = 3; // Host address string host_address = 4; + // Which NIC this mapping's traffic enters through, as an index into + // `networks`. Unset picks the first user-mode NIC, which is where QEMU's + // hostfwd entries have always gone and the only place a host port is + // published from. + // + // A VM with one NIC never needs it. With several there is a choice, and it + // used to be made silently: a bridge NIC for external traffic beside a + // user-mode NIC for management -- the topology multi-NIC was added for -- + // put every published port on the management NIC. + optional uint32 nic_index = 5; } // Partial configuration used when mutating an existing VM. @@ -288,9 +329,17 @@ message ResourcesSettings { message NetworkingCapabilities { repeated string supported_modes = 1; string default_mode = 2; - bool forward_service_enabled = 3; + reserved 3; + reserved "forward_service_enabled"; // Default bridge configured in vmm.toml [cvm.networking].bridge. string default_bridge = 4; + // Largest virtio-net queue pair count a deployment request may ask for. + uint32 max_queues = 5; + // Whether the node's own backend runs the vhost-net data plane. A NIC that + // pins neither vhost nor a queue count follows this, and without vhost the + // queue count does not scale with vCPUs -- so a client cannot describe what + // an empty queue field will do without it. + bool default_vhost = 6; } // Aggregated metadata exposed through GetMeta. @@ -372,10 +421,6 @@ service Vmm { // Reload VMs directory and sync with memory state rpc ReloadVms(google.protobuf.Empty) returns (ReloadVmsResponse); - // Report a DHCP lease event (called by the DHCP server, e.g. dnsmasq --dhcp-script). - // The VMM resolves the MAC address to a VM and reconfigures port forwarding. - rpc ReportDhcpLease(DhcpLeaseRequest) returns (google.protobuf.Empty); - // List all supervisor processes. rpc SvList(google.protobuf.Empty) returns (SvListResponse); // Stop a supervisor process by ID. @@ -391,14 +436,6 @@ service Vmm { rpc DeleteImage(Id) returns (google.protobuf.Empty); } -// DHCP lease event reported by the host DHCP server. -message DhcpLeaseRequest { - // MAC address of the guest NIC (e.g. "02:ab:cd:ef:01:23") - string mac = 1; - // IPv4 address assigned by DHCP (e.g. "192.168.122.100") - string ip = 2; -} - // Response containing a list of supervisor processes. message SvListResponse { repeated SvProcessInfo processes = 1; diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 2ab3aeee9..1675aec25 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -2,58 +2,106 @@ // // SPDX-License-Identifier: Apache-2.0 -use crate::config::{Config, Networking, ProcessAnnotation, Protocol}; -use dstack_port_forward::{ForwardRule, ForwardService, Protocol as FwdProtocol}; +use crate::{ + config::{Config, Networking, NetworkingMode, NicNetworking, ProcessAnnotation, Protocol}, + logrotate, + netd::{self, InterfaceIdentity, PrepareBridgeRequest, PrepareMacvtapRequest}, +}; use anyhow::{bail, Context, Result}; use bon::Builder; use dstack_kms_rpc::kms_client::KmsClient; use dstack_types::mr_config::MrConfigV3; use dstack_types::shared_filenames::{ - APP_COMPOSE, ENCRYPTED_ENV, INSTANCE_INFO, SYS_CONFIG, USER_CONFIG, + APP_COMPOSE, ENCRYPTED_ENV, INSTANCE_INFO, SYS_CONFIG, TEE_SIMULATOR_CONFIG, USER_CONFIG, }; +use dstack_types::version::Version; use dstack_vmm_rpc::{ self as pb, GpuInfo, ReloadVmsResponse, StatusRequest, StatusResponse, VmConfiguration, }; use fs_err as fs; use guest_api::client::DefaultClient as GuestClient; use id_pool::IdPool; +use nix::unistd::Uid; use or_panic::ResultOrPanic; use ra_rpc::client::RaClient; +use rand::seq::SliceRandom; use serde::{Deserialize, Serialize}; use serde_json::json; use sha2::{Digest, Sha256}; -use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; use std::net::IpAddr; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, MutexGuard}; -use std::time::SystemTime; +use std::time::{Duration, SystemTime}; use supervisor_client::SupervisorClient; use tracing::{debug, error, info, warn}; pub use image::{Image, ImageInfo}; pub(crate) use network::{ - resolve_networking, resolved_networks, validate_resolved_network, validate_resolved_networks, + filters_bridge_traffic, mode_carries_ingress, needs_netd_interface, resolve_networking, + resolved_networks, settle_vhost, stranded_ingress, validate_resolved_network, + validate_resolved_networks, }; pub use qemu::VmConfig; +// Exported so the RPC layer can assert that everything it reports is +// something it also accepts. +#[cfg(test)] +pub(crate) use vm_info::networking_to_proto; pub use workdir::VmWorkDir; mod host_share; mod id_pool; mod image; mod mr_config; -mod network; +pub(crate) mod network; mod qemu; pub(crate) mod registry; mod vm_info; mod workdir; +fn signal_pidfd(pid: u32, signal: libc::c_int) -> std::io::Result<()> { + // SAFETY: pidfd syscalls receive scalar arguments and a null siginfo pointer. + let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) }; + if fd < 0 { + return Err(std::io::Error::last_os_error()); + } + let result = unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + fd, + signal, + std::ptr::null::<libc::siginfo_t>(), + 0, + ) + }; + // SAFETY: fd was returned by pidfd_open and is owned by this function. + unsafe { libc::close(fd as libc::c_int) }; + if result < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + #[derive(Deserialize, Serialize, Debug, Clone)] pub struct PortMapping { pub address: IpAddr, pub protocol: Protocol, pub from: u16, pub to: u16, + /// Which NIC carries this mapping. `None` resolves by the node's rule; see + /// [`crate::app::network::ingress_nic`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub nic_index: Option<usize>, +} + +/// An extra disk attached to the VM (e.g. a pre-baked verity volume). `source` +/// is an absolute host path already resolved and validated by the VMM against +/// `cvm.volumes_dir`. +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct VmVolume { + pub source: String, } #[derive(Deserialize, Serialize, Clone, Builder, Debug)] @@ -79,8 +127,15 @@ pub struct Manifest { pub gateway_urls: Vec<String>, #[serde(default)] pub no_tee: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub simulated_tee: Option<dstack_types::TeeVariant>, + /// Deployment-time decision to attach QEMU swtpm. + #[serde(default)] + pub swtpm: bool, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub networks: Vec<Networking>, + pub networks: Vec<NicNetworking>, + #[serde(default)] + pub volumes: Vec<VmVolume>, } impl Manifest { @@ -132,6 +187,10 @@ pub struct GpuConfig { } impl GpuConfig { + pub fn has_gpus(&self) -> bool { + !self.gpus.is_empty() + } + pub fn is_empty(&self) -> bool { if self.attach_mode.is_all() { return false; @@ -246,11 +305,12 @@ pub struct App { pub config: Arc<Config>, pub supervisor: SupervisorClient, state: Arc<Mutex<AppState>>, - forward_service: Arc<tokio::sync::Mutex<ForwardService>>, /// Pull status for registry images: tag → status. pub(crate) pull_status: Arc<Mutex<std::collections::HashMap<String, PullStatus>>>, } +const GUEST_AGENT_RPC_TIMEOUT: Duration = Duration::from_secs(30); + impl App { pub(crate) fn lock(&self) -> MutexGuard<'_, AppState> { self.state.lock().or_panic("mutex poisoned") @@ -260,8 +320,9 @@ impl App { self.config.run_path.clone() } - pub(crate) fn work_dir(&self, id: &str) -> VmWorkDir { - VmWorkDir::new(self.config.run_path.join(id)) + pub(crate) fn work_dir(&self, id: &str) -> Result<VmWorkDir> { + validate_vm_id(id)?; + Ok(VmWorkDir::new(self.config.run_path.join(id))) } pub fn new(config: Config, supervisor: SupervisorClient) -> Self { @@ -273,10 +334,8 @@ impl App { state: Arc::new(Mutex::new(AppState { cid_pool, vms: HashMap::new(), - active_forwards: HashMap::new(), })), config: Arc::new(config), - forward_service: Arc::new(tokio::sync::Mutex::new(ForwardService::new())), pull_status: Arc::new(Mutex::new(std::collections::HashMap::new())), } } @@ -303,7 +362,7 @@ impl App { let vm_id = manifest.id.clone(); let mut runtime_networks = vm_work_dir.runtime_networks(); if runtime_networks.is_empty() && cids_assigned.contains_key(&vm_id) { - runtime_networks = resolved_networks(&manifest, &self.config.cvm); + runtime_networks = self.inferred_runtime_networks(&manifest); if let Err(err) = vm_work_dir.set_runtime_networks(&runtime_networks) { warn!(id = %vm_id, "failed to persist inferred runtime networks: {err}"); } @@ -345,6 +404,19 @@ impl App { } pub async fn start_vm(&self, id: &str) -> Result<()> { + self.start_vm_with_restart_policy(id, true).await + } + + async fn start_vm_with_restart_policy( + &self, + id: &str, + reset_restart_policy: bool, + ) -> Result<()> { + if reset_restart_policy { + if let Some(vm) = self.lock().get_mut(id) { + vm.state.auto_restart.reset(); + } + } { let state = self.lock(); if let Some(vm) = state.get(id) { @@ -372,29 +444,72 @@ impl App { vm_state.config.clone() }; if !is_running { - let work_dir = self.work_dir(id); + let work_dir = self.work_dir(id)?; for path in [work_dir.serial_pty(), work_dir.qmp_socket()] { if path.symlink_metadata().is_ok() { fs::remove_file(path)?; } } - // Append current serial.log to serial.history.log before QEMU truncates it. - rotate_serial_log(&work_dir, self.config.cvm.serial_history_max_bytes); - // Add boot separator to stdout/stderr (they are opened in append mode). - append_boot_separator(&work_dir.stdout_file()); - append_boot_separator(&work_dir.stderr_file()); + // Archive the previous boot into segments, which also clears the + // live logs for this boot. QEMU runs with logappend=on and no + // longer truncates serial.log on open, so a boot is simply another + // rotation trigger and boot boundaries land on segment boundaries. + for path in rotatable_logs(&work_dir, true) { + rotate_log(&path, self.config.cvm.log.max_backups); + // The logs are opened in append mode, so this marks the start + // of the new boot rather than replacing anything. + append_boot_separator(&path); + } + let mut runtime_networks = self.runtime_networks(&vm_config.manifest); let devices = self.try_allocate_gpus(&vm_config.manifest)?; - let processes = vm_config.config_qemu(&work_dir, &self.config.cvm, &devices)?; - let runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm); - work_dir.set_runtime_networks(&runtime_networks)?; + let gpu_host_config = self.config.cvm.gpu.clone(); + let devices_to_sanitize = devices.clone(); + tokio::task::spawn_blocking(move || { + crate::gpu_reset::sanitize_on_attach(&gpu_host_config, &devices_to_sanitize) + }) + .await + .context("GPU sanitization task failed")??; + if let Err(error) = self + .prepare_netd_networks(&vm_config, &mut runtime_networks) + .await + { + let _ = work_dir.clear_runtime_networks(); + return Err(error); + } + let processes = match vm_config.config_qemu( + &work_dir, + &self.config.cvm, + &devices, + &runtime_networks, + ) { + Ok(processes) => processes, + Err(error) => { + let _ = self + .remove_netd_networks(&vm_config.manifest.id, &runtime_networks) + .await; + return Err(error); + } + }; + if let Err(error) = work_dir.set_runtime_networks(&runtime_networks) { + let _ = self + .remove_netd_networks(&vm_config.manifest.id, &runtime_networks) + .await; + return Err(error); + } { let mut state = self.lock(); let vm_state = state.get_mut(id).context("VM not found")?; - vm_state.state.runtime_networks = runtime_networks; + vm_state.state.runtime_networks = runtime_networks.clone(); } for process in processes { if let Err(err) = self.supervisor.deploy(&process).await { + if let Err(cleanup_error) = self + .remove_netd_networks(&vm_config.manifest.id, &runtime_networks) + .await + { + warn!(id, %cleanup_error, "failed to roll back netd-managed networking"); + } if let Err(clear_err) = work_dir.clear_runtime_networks() { warn!( id, @@ -417,19 +532,279 @@ impl App { } fn set_started(&self, id: &str, started: bool) -> Result<()> { - let work_dir = self.work_dir(id); + let work_dir = self.work_dir(id)?; work_dir .set_started(started) .context("Failed to set started") } pub async fn stop_vm(&self, id: &str) -> Result<()> { + if let Some(vm) = self.lock().get_mut(id) { + vm.state.auto_restart.reset(); + } self.set_started(id, false)?; - self.cleanup_port_forward(id).await; - self.supervisor.stop(id).await?; + self.stop_vm_process(id).await?; + let networks = self.work_dir(id)?.runtime_networks(); + self.remove_netd_networks(id, &networks).await?; Ok(()) } + async fn prepare_netd_networks( + &self, + vm: &VmConfig, + networks: &mut [Networking], + ) -> Result<()> { + // Before the early return, because a mapping with nowhere to go is a + // property of the resolved topology and not of whether netd is in it. + // Deployment refuses every way of asking for one, so reaching this + // means an edit removed the NIC out from under a mapping that named it. + for mapping in stranded_ingress(&vm.manifest.port_map, networks) { + warn!( + vm_id = %vm.manifest.id, + "port mapping {} {}:{} names NIC {:?}, which this VM no longer has a backend \ + for; it will not be published", + mapping.protocol.as_str(), + mapping.address, + mapping.from, + mapping.nic_index, + ); + } + if !networks.iter().any(needs_netd_interface) { + return Ok(()); + } + let qemu_uid = Uid::effective().as_raw(); + let mut prepared = Vec::new(); + for (nic_index, network) in networks.iter_mut().enumerate() { + if !needs_netd_interface(network) { + continue; + } + let identity = InterfaceIdentity { + instance_id: self.config.cvm.instance_id.clone(), + vm_id: vm.manifest.id.clone(), + nic_index: nic_index as u32, + }; + let mac = network::mac_address_for_vm_index( + &vm.manifest.id, + &network.mac_prefix_bytes(), + nic_index, + ); + let queues = network.queue_pairs(); + let filtered = filters_bridge_traffic(network, &self.config.cvm); + let netd = netd::client(&self.config.netd.socket); + let result = match network.nic.mode { + NetworkingMode::Bridge => { + netd.prepare_bridge(PrepareBridgeRequest { + identity: Some(identity.clone()), + bridge: network.nic.bridge.clone(), + mac, + qemu_uid, + // Which filter, and with what parameters, is netd's + // to decide from its own configuration. An unfiltered + // TAP is only asked for by multiqueue, where the node + // may not run libvirt at all. + filtered, + queues, + }) + .await + } + NetworkingMode::Macvtap => { + netd.prepare_macvtap(PrepareMacvtapRequest { + identity: Some(identity.clone()), + parent: network.nic.parent.clone(), + mac, + qemu_uid, + mode: network.macvtap_mode.clone(), + queues, + }) + .await + } + NetworkingMode::User | NetworkingMode::Custom => continue, + }; + let response = match result { + Ok(response) => response, + Err(error) => { + // The client may have timed out while netd was still + // finishing this Prepare. Remove the in-flight identity + // first; netd runs operations in arrival order, so the + // removal runs after the Prepare completes. + if let Err(cleanup_error) = netd.remove_interface(identity.clone()).await { + warn!(%cleanup_error, "failed to roll back in-flight netd network"); + } + self.roll_back_prepared_networks(prepared).await; + // netd's own message is about a socket or a TAP, so + // neither the NIC that asked nor what a node has to install + // to satisfy it appears anywhere in the failure. + let unreachable = netd::is_unreachable(&error); + let mode = network.nic.mode.as_str(); + let error = Err(error).context("failed to prepare netd-managed networking"); + return if unreachable { + error.with_context(|| { + format!( + "interface {nic_index} is {mode}, whose host interface only netd \ + can build; run dstack-vmm netd on this host" + ) + }) + } else if queues > 1 { + error.with_context(|| { + format!("interface {nic_index} asked for {queues} queue pairs") + }) + } else { + error.with_context(|| format!("interface {nic_index} is {mode}")) + }; + } + }; + prepared.push(identity.clone()); + // Everything below runs after netd already built a host interface, + // so a failure has to unwind the same way a failed Prepare does. + let accepted = (|| { + if network.nic.mode == NetworkingMode::Macvtap { + if response.device.is_empty() { + bail!("netd response omitted macvtap device"); + } + network.device = response.device.clone(); + } + // QEMU refuses a TAP whose IFF_MULTI_QUEUE state disagrees with + // its own `queues=`, and reports it from inside the per-VM + // launcher. netd echoes what it built, so a mismatch fails + // here, where the reason is legible. + if queues > 1 && response.queues != queues { + bail!( + "netd prepared interface {nic_index} with {} queue pairs instead of \ + {queues}", + response.queues + ); + } + Ok(()) + })(); + if let Err(error) = accepted { + self.roll_back_prepared_networks(prepared).await; + return Err(error); + } + } + Ok(()) + } + + /// The NICs a VM has now, or would get if it were started. + /// + /// While QEMU is up this is what the launch actually built. Once it is + /// down the snapshot describes a boot that is over: the node configuration + /// and the VM's own manifest can both have changed since, so reporting it + /// would answer a question about the past with the grammar of the present. + /// Predict instead, the same way the next launch will. + fn effective_networks(&self, info: &vm_info::VmInfo) -> Vec<Networking> { + if info.running && !info.runtime_networks.is_empty() { + return info.runtime_networks.clone(); + } + self.merge_networks(&info.manifest) + } + + /// Launch-time view of a VM's NICs: node defaults merged in and the + /// vCPU-scaled queue count made concrete. + pub(crate) fn runtime_networks(&self, manifest: &Manifest) -> Vec<Networking> { + self.merge_networks(manifest) + } + + /// A running VM whose snapshot is missing, because a VMM that predates the + /// snapshot started it. + /// + /// Guessing is all that is left, so guess the way that VMM would have, and + /// then write the guess down, so later teardown does not re-derive it from + /// node configuration that may by then have moved. + /// + /// The way *that* VMM would have, not this one: a build old enough to leave + /// no snapshot had no vhost and no multiqueue at all, so whatever this + /// node's defaults say now, the QEMU process actually running was given one + /// queue pair and no vhost. Asking `runtime_networks` would apply today's + /// defaults to a launch that predates them, and the guess is persisted, so + /// it would keep describing that VM wrongly for the life of its boot. + fn inferred_runtime_networks(&self, manifest: &Manifest) -> Vec<Networking> { + let mut networks = self.merge_networks(manifest); + for network in &mut networks { + network.nic.vhost = Some(false); + network.nic.queues = Some(1); + } + networks + } + + /// The merge itself: node defaults applied, then the data plane settled so + /// that every later stage reads one answer instead of recomputing it. + fn merge_networks(&self, manifest: &Manifest) -> Vec<Networking> { + let mut resolved = resolved_networks(manifest, &self.config.cvm); + settle_vhost(&mut resolved); + resolved + } + + /// Removes interfaces netd already built for a launch that then failed. + async fn roll_back_prepared_networks(&self, prepared: Vec<InterfaceIdentity>) { + let netd = netd::client(&self.config.netd.socket); + for identity in prepared.into_iter().rev() { + if let Err(cleanup_error) = netd.remove_interface(identity).await { + warn!(%cleanup_error, "failed to roll back prepared network interface"); + } + } + } + + /// Removes the host interfaces netd built for the NICs in `networks`. + pub(crate) async fn remove_netd_networks( + &self, + vm_id: &str, + networks: &[Networking], + ) -> Result<()> { + let netd = netd::client(&self.config.netd.socket); + let mut first_error = None; + for (nic_index, network) in networks.iter().enumerate().rev() { + if !needs_netd_interface(network) { + continue; + } + let identity = InterfaceIdentity { + instance_id: self.config.cvm.instance_id.clone(), + vm_id: vm_id.to_string(), + nic_index: nic_index as u32, + }; + if let Err(error) = netd.remove_interface(identity).await { + first_error.get_or_insert(error); + } + } + if let Some(error) = first_error { + return Err(error).context("failed to remove netd-managed networking"); + } + Ok(()) + } + + pub(crate) async fn stop_vm_process(&self, id: &str) -> Result<()> { + let Some(info) = self.supervisor.info(id).await? else { + return Ok(()); + }; + // Non-TPM VMs run QEMU directly and keep the existing Supervisor stop + // path. Only the TPM launcher's hidden subcommand implements graceful + // child-process shutdown. + if info.config.args.first().map(String::as_str) != Some("vm-launcher") { + return self.supervisor.stop(id).await; + } + if info.state.status.is_running() { + let pid = info.state.pid.context("running VM launcher has no PID")?; + if let Err(error) = signal_pidfd(pid, libc::SIGTERM) { + warn!(id, %pid, %error, "failed to signal VM launcher gracefully; forcing shutdown"); + return self.supervisor.stop(id).await; + } + for _ in 0..150 { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + let running = self + .supervisor + .info(id) + .await? + .is_some_and(|info| info.state.status.is_running()); + if !running { + // Synchronize Supervisor's `started` flag after the launcher + // completed its graceful child cleanup. + return self.supervisor.stop(id).await; + } + } + warn!(id, "VM launcher did not stop gracefully; forcing shutdown"); + } + self.supervisor.stop(id).await + } + pub async fn remove_vm(&self, id: &str) -> Result<()> { { let mut state = self.lock(); @@ -442,14 +817,11 @@ impl App { } // Persist the removing marker so crash recovery can resume - let work_dir = self.work_dir(id); + let work_dir = self.work_dir(id)?; if let Err(err) = work_dir.set_removing() { warn!("failed to write .removing marker for {id}: {err:?}"); } - // Clean up port forwarding immediately - self.cleanup_port_forward(id).await; - // User-initiated removal always deletes the workdir let app = self.clone(); let id = id.to_string(); @@ -468,8 +840,8 @@ impl App { /// `delete_workdir`: true for user-initiated removal, false for orphan cleanup. async fn finish_remove_vm(&self, id: &str, delete_workdir: bool) -> Result<()> { // Stop the supervisor process (idempotent if already stopped) - if let Err(err) = self.supervisor.stop(id).await { - debug!("supervisor.stop({id}) during removal: {err:?}"); + if let Err(err) = self.stop_vm_process(id).await { + debug!("graceful VM stop during removal failed: {err:?}"); } // Poll until the process is no longer running, then remove it. @@ -505,9 +877,14 @@ impl App { } } + let runtime_networks = self.work_dir(id)?.runtime_networks(); + if let Err(error) = self.remove_netd_networks(id, &runtime_networks).await { + warn!(id, %error, "failed to remove netd-managed networking during VM removal"); + } + // Only delete the workdir for user-initiated removal or if .removing marker exists. // Orphaned supervisor processes without the marker keep their data intact. - let vm_path = self.work_dir(id); + let vm_path = self.work_dir(id)?; if delete_workdir || vm_path.is_removing() { if vm_path.path().exists() { if let Err(err) = fs::remove_dir_all(&vm_path) { @@ -560,140 +937,6 @@ impl App { true } - /// Handle a DHCP lease notification: look up VM by MAC address, persist - /// the guest IP, and reconfigure port forwarding. - pub async fn report_dhcp_lease(&self, mac: &str, ip: &str) { - use crate::app::network::mac_address_for_vm_index; - - let vm_id = { - let mut state = self.lock(); - let found = state.vms.iter_mut().find_map(|(id, vm)| { - let networks = vm.effective_networks(&self.config.cvm); - networks.iter().enumerate().find_map(|(index, networking)| { - let prefix = networking.mac_prefix_bytes(); - let nic_mac = mac_address_for_vm_index(&vm.config.manifest.id, &prefix, index); - (nic_mac == mac).then(|| id.clone()) - }) - }); - let Some(vm_id) = found else { - debug!(mac, ip, "DHCP lease for unknown MAC, ignoring"); - return; - }; - let Some(vm) = state.get_mut(&vm_id) else { - debug!(mac, ip, id = %vm_id, "DHCP lease for missing VM, ignoring"); - return; - }; - let workdir = VmWorkDir::new(vm.config.workdir.clone()); - if let Err(e) = workdir.set_guest_ip(ip) { - error!(mac, ip, "failed to persist guest IP: {e}"); - } - vm.state.guest_ip = ip.to_string(); - info!(mac, ip, id = %vm_id, "DHCP lease updated"); - vm_id - }; - self.reconfigure_port_forward(&vm_id).await; - } - - /// Reconfigure port forwarding for a bridge-mode VM. - /// - /// Computes desired rules from the VM's port_map and guest_ip, then diffs - /// against currently active rules. Only changed rules are added/removed so - /// existing connections on unchanged rules are not interrupted. - pub async fn reconfigure_port_forward(&self, id: &str) { - let info = { - let state = self.lock(); - let Some(vm) = state.get(id) else { - return; - }; - let networks = vm.effective_networks(&self.config.cvm); - if networks - .iter() - .any(|networking| networking.mode == crate::config::NetworkingMode::User) - { - return; - } - if !networks - .iter() - .any(|networking| networking.is_bridge() && networking.forward_service_enabled) - { - return; - } - let guest_ip = vm.state.guest_ip.clone(); - let port_map = vm.config.manifest.port_map.clone(); - (guest_ip, port_map) - }; - - let (guest_ip_str, port_map) = info; - if guest_ip_str.is_empty() { - return; - } - let Ok(guest_ip) = guest_ip_str.parse::<IpAddr>() else { - warn!(id, ip = %guest_ip_str, "invalid guest IP, skipping port forward"); - return; - }; - - let new_rules: Vec<ForwardRule> = port_map - .iter() - .map(|pm| ForwardRule { - protocol: match pm.protocol { - Protocol::Tcp => FwdProtocol::Tcp, - Protocol::Udp => FwdProtocol::Udp, - }, - listen_addr: pm.address, - listen_port: pm.from, - target_ip: guest_ip, - target_port: pm.to, - }) - .collect(); - - let old_rules = self - .lock() - .active_forwards - .get(id) - .cloned() - .unwrap_or_default(); - - let old_set: HashSet<_> = old_rules.iter().collect(); - let new_set: HashSet<_> = new_rules.iter().collect(); - - let mut fwd = self.forward_service.lock().await; - - // Remove rules no longer needed - for rule in old_rules.iter().filter(|r| !new_set.contains(r)) { - if let Err(e) = fwd.remove_rule(rule).await { - warn!(id, ?rule, "failed to remove forwarding rule: {e}"); - } - } - - // Add new rules - for rule in new_rules.iter().filter(|r| !old_set.contains(r)) { - if let Err(e) = fwd.add_rule(rule.clone()) { - warn!(id, ?rule, "failed to add forwarding rule: {e}"); - } - } - - drop(fwd); - self.lock() - .active_forwards - .insert(id.to_string(), new_rules); - info!(id, "port forwarding reconfigured"); - } - - /// Remove all port forwarding rules for a VM. - pub async fn cleanup_port_forward(&self, id: &str) { - let old_rules = self.lock().active_forwards.remove(id).unwrap_or_default(); - if old_rules.is_empty() { - return; - } - let mut fwd = self.forward_service.lock().await; - for rule in &old_rules { - if let Err(e) = fwd.remove_rule(rule).await { - warn!(id, ?rule, "failed to remove forwarding rule: {e}"); - } - } - info!(id, count = old_rules.len(), "port forwarding cleaned up"); - } - pub async fn reload_vms(&self) -> Result<()> { let vm_path = self.vm_dir(); let running_vms = self.supervisor.list().await.context("Failed to list VMs")?; @@ -708,8 +951,13 @@ impl App { .collect::<HashMap<_, _>>(); { let mut state = self.lock(); - for cid in occupied_cids.values() { - state.cid_pool.occupy(*cid)?; + for (vm_id, cid) in occupied_cids.iter() { + // These CIDs come from processes that are already running, not + // from the pool, so a CID left over from an earlier cid_start / + // cid_pool_size must not stop the VMM from starting. + if let Err(err) = state.cid_pool.occupy(*cid) { + warn!(id = %vm_id, "not tracking cid {cid} in the pool: {err}"); + } } } @@ -754,21 +1002,6 @@ impl App { } } - // Restore port forwarding for running bridge-mode VMs with persisted guest IPs - let vm_ids: Vec<String> = self.lock().vms.keys().cloned().collect(); - for id in vm_ids { - let workdir = self.work_dir(&id); - if let Some(ip) = workdir.guest_ip() { - { - let mut state = self.lock(); - if let Some(vm) = state.get_mut(&id) { - vm.state.guest_ip = ip; - } - } - self.reconfigure_port_forward(&id).await; - } - } - Ok(()) } @@ -795,13 +1028,20 @@ impl App { .flat_map(|(id, p)| p.config.cid.map(|cid| (id.clone(), cid))) .collect::<HashMap<_, _>>(); - // Update CID pool with running VMs + // Rebuild the CID pool from every CID that is still spoken for { let mut state = self.lock(); - // First clear the pool and re-occupy running VM CIDs + let reserved = cids_to_reserve( + &occupied_cids, + state.vms.iter().map(|(id, vm)| (id.clone(), vm.config.cid)), + ); state.cid_pool.clear(); - for cid in occupied_cids.values() { - state.cid_pool.occupy(*cid)?; + for (cid, vm_id) in reserved { + // Same as in reload_vms(): an out-of-range CID from an earlier + // configuration is reported, not fatal. + if let Err(err) = state.cid_pool.occupy(cid) { + warn!(id = %vm_id, "not tracking cid {cid} in the pool: {err}"); + } } } @@ -919,7 +1159,7 @@ impl App { let already_running = cids_assigned.contains_key(&vm_id); let mut runtime_networks = vm_work_dir.runtime_networks(); if runtime_networks.is_empty() && already_running { - runtime_networks = resolved_networks(&manifest, &self.config.cvm); + runtime_networks = self.inferred_runtime_networks(&manifest); if let Err(err) = vm_work_dir.set_runtime_networks(&runtime_networks) { warn!(id = %vm_id, "failed to persist inferred runtime networks: {err}"); } @@ -964,10 +1204,7 @@ impl App { vm.state = old_state; // Preserve the existing state with statistics } None => { - // This is a new VM, need to occupy its CID if it wasn't allocated - if !cids_assigned.contains_key(&vm_id) { - states.cid_pool.occupy(cid)?; - } + // Assigned CIDs were occupied above, while allocate() reserves a new CID. let mut vm_state = VmState::new(vm_config); vm_state.state.runtime_networks = runtime_networks; states.add(vm_state); @@ -1025,13 +1262,12 @@ impl App { let total = infos.len() as u32; let vms = paginate(infos, request.page, request.page_size) .map(|vm| { - vm.merged_info( - vms.get(&vm.config.manifest.id), - &self.work_dir(&vm.config.manifest.id), - ) + let work_dir = self.work_dir(&vm.config.manifest.id)?; + let info = vm.merged_info(vms.get(&vm.config.manifest.id), &work_dir); + let networks = self.effective_networks(&info); + Ok(info.to_pb(&self.config.gateway, request.brief, &networks)) }) - .map(|info| info.to_pb(&self.config.gateway, &self.config.cvm, request.brief)) - .collect::<Vec<_>>(); + .collect::<Result<Vec<_>>>()?; Ok(StatusResponse { vms, port_mapping_enabled: self.config.cvm.port_mapping.enabled, @@ -1053,14 +1289,18 @@ impl App { pub async fn vm_info(&self, id: &str) -> Result<Option<pb::VmInfo>> { let proc_state = self.supervisor.info(id).await?; - let state = self.lock(); - let Some(vm_state) = state.get(id) else { - return Ok(None); + // Snapshot under the lock, then release it: the global state lock is + // held by every other VM's operations, and describing one VM has no + // business keeping it across the work that follows. + let info = { + let state = self.lock(); + let Some(vm_state) = state.get(id) else { + return Ok(None); + }; + vm_state.merged_info(proc_state.as_ref(), &self.work_dir(id)?) }; - let info = vm_state - .merged_info(proc_state.as_ref(), &self.work_dir(id)) - .to_pb(&self.config.gateway, &self.config.cvm, false); - Ok(Some(info)) + let networks = self.effective_networks(&info); + Ok(Some(info.to_pb(&self.config.gateway, false, &networks))) } pub(crate) fn vm_event_report(&self, cid: u32, event: &str, body: String) -> Result<()> { @@ -1109,20 +1349,21 @@ impl App { Ok(()) } - pub(crate) fn compose_file_path(&self, id: &str) -> PathBuf { - self.shared_dir(id).join(APP_COMPOSE) + pub(crate) fn compose_file_path(&self, id: &str) -> Result<PathBuf> { + Ok(self.shared_dir(id)?.join(APP_COMPOSE)) } - pub(crate) fn encrypted_env_path(&self, id: &str) -> PathBuf { - self.shared_dir(id).join(ENCRYPTED_ENV) + pub(crate) fn encrypted_env_path(&self, id: &str) -> Result<PathBuf> { + Ok(self.shared_dir(id)?.join(ENCRYPTED_ENV)) } - pub(crate) fn user_config_path(&self, id: &str) -> PathBuf { - self.shared_dir(id).join(USER_CONFIG) + pub(crate) fn user_config_path(&self, id: &str) -> Result<PathBuf> { + Ok(self.shared_dir(id)?.join(USER_CONFIG)) } - pub(crate) fn shared_dir(&self, id: &str) -> PathBuf { - self.config.run_path.join(id).join("shared") + pub(crate) fn shared_dir(&self, id: &str) -> Result<PathBuf> { + validate_vm_id(id)?; + Ok(self.config.run_path.join(id).join("shared")) } pub(crate) fn prepare_work_dir( @@ -1131,7 +1372,7 @@ impl App { req: &VmConfiguration, app_id: &str, ) -> Result<VmWorkDir> { - let work_dir = self.work_dir(id); + let work_dir = self.work_dir(id)?; let shared_dir = work_dir.join("shared"); fs::create_dir_all(&shared_dir).context("Failed to create shared directory")?; fs::write(shared_dir.join(APP_COMPOSE), &req.compose_file) @@ -1158,29 +1399,17 @@ impl App { } pub(crate) fn sync_dynamic_config(&self, id: &str) -> Result<()> { - let work_dir = self.work_dir(id); - let shared_dir = self.shared_dir(id); + let work_dir = self.work_dir(id)?; + let shared_dir = self.shared_dir(id)?; let manifest = work_dir.manifest().context("Failed to read manifest")?; let cfg = &self.config; let compose_hash = sha256_file(shared_dir.join(APP_COMPOSE))?; - let platform = cfg.cvm.resolved_platform(); let app_compose = work_dir .app_compose() .context("Failed to get app compose")?; - let use_mr_config_v3 = !manifest.no_tee - && (platform == crate::config::TeePlatform::AmdSevSnp - || (platform == crate::config::TeePlatform::Tdx - && cfg.cvm.use_mrconfigid - && !app_compose.key_provider_id.is_empty())); - let mr_config = if use_mr_config_v3 { - Some( - work_dir - .prepare_mr_config_v3(&app_compose) - .context("Failed to prepare mr_config")?, - ) - } else { - None - }; + let mr_config = work_dir + .prepare_mr_config(&manifest, &cfg.cvm, &app_compose) + .context("Failed to prepare mr_config")?; let sys_config_str = make_sys_config( cfg, &manifest, @@ -1188,8 +1417,10 @@ impl App { mr_config, app_compose.requirements.as_ref(), )?; - fs::write(shared_dir.join(SYS_CONFIG), sys_config_str) + fs::write(shared_dir.join(SYS_CONFIG), &sys_config_str) .context("Failed to write vm config")?; + let simulator_config = simulator_config_for_manifest(&self.config.cvm, &manifest)?; + sync_tee_simulator_config(&shared_dir, simulator_config.as_ref(), &sys_config_str)?; Ok(()) } @@ -1204,9 +1435,10 @@ impl App { pub(crate) fn guest_agent_client(&self, id: &str) -> Result<GuestClient> { let cid = self.lock().get(id).context("vm not found")?.config.cid; - Ok(guest_api::client::new_client(format!( - "vsock://{cid}:8000/api" - ))) + Ok(guest_api::client::new_client_with_timeout( + format!("vsock://{cid}:8000/api"), + GUEST_AGENT_RPC_TIMEOUT, + )) } fn try_allocate_gpus(&self, manifest: &Manifest) -> Result<GpuConfig> { @@ -1236,6 +1468,41 @@ impl App { Ok(gpus) } + /// Rotate any live log that has grown past the configured cap. + pub(crate) async fn rotate_oversized_logs(&self) -> Result<()> { + let max_bytes = self.config.cvm.log.max_bytes; + if max_bytes == 0 { + return Ok(()); + } + let max_backups = self.config.cvm.log.max_backups; + let running = self + .supervisor + .list() + .await + .context("failed to list VMs")? + .into_iter() + .filter(|process| process.state.status.is_running()); + for process in running { + let Ok(work_dir) = self.work_dir(&process.config.id) else { + continue; + }; + let serial = serial_log_is_rotatable(&process.config.note); + for path in rotatable_logs(&work_dir, serial) { + if let Some(rotated) = logrotate::rotate_if_oversized(&path, max_bytes, max_backups) + { + logrotate::append_rotation_note(&path, &rotated); + info!( + id = process.config.id, + log = %path.display(), + bytes = rotated.bytes, + "rotated oversized log" + ); + } + } + } + Ok(()) + } + pub(crate) async fn try_restart_exited_vms(&self) -> Result<()> { let running_vms = self .supervisor @@ -1246,83 +1513,193 @@ impl App { .filter(|v| v.state.status.is_running()) .map(|v| v.config.id.clone()) .collect::<BTreeSet<_>>(); - let exited_vms = self - .lock() - .iter_vms() - .filter(|vm| { + let now = std::time::Instant::now(); + let mut restart_vms = Vec::new(); + { + let mut state = self.lock(); + for vm in state.vms.values_mut() { + let id = &vm.config.manifest.id; if vm.state.removing { - return false; + vm.state.auto_restart.reset(); + continue; } - let workdir = self.work_dir(&vm.config.manifest.id); + if running_vms.contains(id) { + if vm + .state + .auto_restart + .observe_running(now, self.config.cvm.auto_restart.reset_window) + { + info!( + id, + "automatic restart retry budget reset after healthy window" + ); + } + continue; + } + let Ok(workdir) = self.work_dir(id) else { + warn!(id, "skipping restart: invalid VM id"); + vm.state.auto_restart.reset(); + continue; + }; let started = workdir.started().unwrap_or(false); - started && !running_vms.contains(&vm.config.manifest.id) - }) - .map(|vm| vm.config.manifest.id.clone()) - .collect::<Vec<_>>(); - for id in exited_vms { - info!("Restarting VM {id}"); - self.start_vm(&id).await?; + if !started { + vm.state.auto_restart.reset(); + continue; + } + match vm + .state + .auto_restart + .observe_exited(now, &self.config.cvm.auto_restart) + { + AutoRestartDecision::Scheduled { delay_secs } => { + info!(id, delay_secs, "automatic restart scheduled"); + } + AutoRestartDecision::Restart { + attempt, + next_delay_secs, + } => { + info!(id, attempt, next_delay_secs, "automatic restart attempt"); + restart_vms.push(id.clone()); + } + AutoRestartDecision::Exhausted { attempts } => { + warn!(id, attempts, "automatic restart retry limit exhausted"); + vm.state.events.push_back(pb::GuestEvent { + event: "vmm.auto_restart.exhausted".into(), + body: format!( + "Automatic restart stopped after {attempts} failed attempts" + ), + timestamp: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64, + }); + while vm.state.events.len() > self.config.event_buffer_size { + vm.state.events.pop_front(); + } + } + AutoRestartDecision::Wait => {} + } + } + } + for id in restart_vms { + // A manual stop may have landed after the restart decision was made. + let Ok(workdir) = self.work_dir(&id) else { + warn!(id, "skipping restart: invalid VM id"); + continue; + }; + if !workdir.started().unwrap_or(false) { + continue; + } + if let Err(error) = self.start_vm_with_restart_policy(&id, false).await { + warn!(id, %error, "automatic restart attempt failed"); + } } Ok(()) } } +/// Leading bytes of the separator written by [`append_boot_separator`]. +/// +/// Written to stdout, stderr and the serial log at each boot so a log read in +/// isolation still shows where a boot began. +const BOOT_SEPARATOR_PREFIX: &str = "\n===== boot @ "; + /// Append a boot separator line with timestamp to an append-mode log file. fn append_boot_separator(path: &std::path::Path) { use std::io::Write; if !path.exists() { return; } - let Ok(mut file) = std::fs::OpenOptions::new().append(true).open(path) else { + let Ok(mut file) = fs::OpenOptions::new().append(true).open(path) else { return; }; let timestamp = humantime::format_rfc3339_seconds(std::time::SystemTime::now()); - let _ = writeln!(file, "\n===== boot @ {timestamp} =====\n"); + let _ = writeln!(file, "{BOOT_SEPARATOR_PREFIX}{timestamp} =====\n"); } -/// Append current serial.log into serial.history.log with a boot separator, -/// then truncate history if it exceeds `max_bytes`. -fn rotate_serial_log(work_dir: &VmWorkDir, max_bytes: u64) { - use std::io::Write; +/// Logs a CVM writes into its work directory, subject to retention. +/// +/// stdout and stderr are written by the supervisor, which always opens them +/// with `append(true)` and reopens them when they change, so they satisfy +/// [`crate::logrotate`]'s contract no matter which VMM launched the VM. +/// serial.log is written by QEMU, whose fd only appends when *we* passed +/// `logappend=on`, so it is included only when `serial` says so. +fn rotatable_logs(work_dir: &VmWorkDir, serial: bool) -> Vec<PathBuf> { + let mut paths = vec![work_dir.stdout_file(), work_dir.stderr_file()]; + if serial { + paths.push(work_dir.serial_file()); + } + paths +} - let serial = work_dir.serial_file(); - if !serial.exists() { - return; - } - let Ok(content) = fs::read(&serial) else { - return; - }; - if content.is_empty() { - return; +/// Rotate a log and record where its output went. +/// +/// The rotation itself is generic (see [`crate::logrotate`]); the note is here +/// because the VMM log API serves only the live file. +fn rotate_log(path: &Path, max_backups: usize) { + if let Some(rotated) = logrotate::rotate(path, max_backups) { + logrotate::append_rotation_note(path, &rotated); } - let history = work_dir.serial_history_file(); - let Ok(mut file) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&history) - else { - return; +} + +/// Whether a supervised process's serial log may be rotated in place. +/// +/// Rotation truncates the log while QEMU holds it open, which only works when +/// QEMU opened it with `logappend=on`. Anything we cannot positively confirm — +/// an annotation from an older VMM, an unparseable note — answers `false`. +fn serial_log_is_rotatable(note: &str) -> bool { + serde_json::from_str::<ProcessAnnotation>(note) + .unwrap_or_default() + .serial_logappend +} + +pub(crate) fn simulator_config_for_manifest( + cvm: &crate::config::CvmConfig, + manifest: &Manifest, +) -> Result<Option<dstack_types::TeeSimulatorConfig>> { + let Some(platform) = manifest.simulated_tee else { + return Ok(None); }; - let timestamp = humantime::format_rfc3339_seconds(std::time::SystemTime::now()); - let _ = writeln!(file, "\n===== boot @ {timestamp} =====\n"); - let _ = file.write_all(&content); - drop(file); - - // Truncate from the front if history exceeds max_bytes. - if let Ok(meta) = fs::metadata(&history) { - if meta.len() > max_bytes { - if let Ok(data) = fs::read(&history) { - let skip = data.len() - max_bytes as usize; - // Find the next newline after skip point to avoid cutting mid-line. - let start = data[skip..] - .iter() - .position(|&b| b == b'\n') - .map(|p| skip + p + 1) - .unwrap_or(skip); - let _ = fs::write(&history, &data[start..]); - } + let mut config = cvm + .tee_simulator + .clone() + .context("tee simulator credentials are not configured on this VMM")?; + config.platform = platform; + Ok(Some(config)) +} + +pub(crate) fn sync_tee_simulator_config( + shared_dir: &Path, + simulator_config: Option<&dstack_types::TeeSimulatorConfig>, + sys_config: &str, +) -> Result<()> { + let path = shared_dir.join(TEE_SIMULATOR_CONFIG); + let Some(simulator_config) = simulator_config else { + if path.exists() { + fs::remove_file(&path).context("failed to remove stale TEE simulator config")?; } - } + return Ok(()); + }; + + let sys_config: dstack_types::SysConfig = serde_json::from_str(sys_config)?; + let mut simulator_config = simulator_config.clone(); + simulator_config.mr_config = sys_config.mr_config; + let vm_config_value: serde_json::Value = serde_json::from_str(&sys_config.vm_config)?; + simulator_config.aws_pcr_replay = vm_config_value + .get("aws_pcr_replay") + .cloned() + .map(serde_json::from_value) + .transpose() + .context("invalid aws_pcr_replay in vm_config")?; + simulator_config.gcp_tpm_replay = vm_config_value + .get("gcp_tpm_replay") + .cloned() + .map(serde_json::from_value) + .transpose() + .context("invalid gcp_tpm_replay in vm_config")?; + simulator_config.vm_config = Some(sys_config.vm_config); + fs::write(path, serde_json::to_vec(&simulator_config)?) + .context("failed to write TEE simulator config") } pub(crate) fn make_sys_config( @@ -1334,19 +1711,35 @@ pub(crate) fn make_sys_config( ) -> Result<String> { let image_path = cfg.image.path.join(&manifest.image); let image = Image::load(image_path).context("Failed to load image info")?; - let img_ver = image.info.version_tuple().unwrap_or((0, 0, 0)); - let kms_urls = if manifest.kms_urls.is_empty() { + let img_ver = image + .info + .version() + .with_context(|| format!("Unparseable image version: {:?}", image.info.version))?; + let mut kms_urls = if manifest.kms_urls.is_empty() { cfg.cvm.kms_urls.clone() } else { manifest.kms_urls.clone() }; - let gateway_urls = if manifest.gateway_urls.is_empty() { + if cfg.cvm.shuffle_kms_urls { + kms_urls.shuffle(&mut rand::thread_rng()); + } + let mut gateway_urls = if manifest.gateway_urls.is_empty() { cfg.cvm.gateway_urls.clone() } else { manifest.gateway_urls.clone() }; - if img_ver < (0, 5, 0) { - bail!("Unsupported image version: {img_ver:?}"); + let mut gateway_clusters = cfg.cvm.gateway_clusters.clone(); + if cfg.cvm.shuffle_gateway_urls { + // Give each CVM an independent starting gateway. The guest preserves + // these orders when it has no previously successful gateway to prefer. + let mut rng = rand::thread_rng(); + gateway_urls.shuffle(&mut rng); + for cluster in &mut gateway_clusters { + cluster.urls.shuffle(&mut rng); + } + } + if img_ver < Version::new(0, 5, 0) { + bail!("Unsupported image version: {img_ver}"); } let vm_config = make_vm_config( @@ -1360,11 +1753,19 @@ pub(crate) fn make_sys_config( let mut sys_config = json!({ "kms_urls": kms_urls, "gateway_urls": gateway_urls, + "gateway_clusters": gateway_clusters, "pccs_url": cfg.cvm.pccs_url, + "collateral_urls": { "pccs": cfg.cvm.pccs_url }, + "nvidia_attestation_proxy_url": cfg.cvm.nvidia_attestation_proxy_url, "docker_registry": cfg.cvm.docker_registry, "host_api_url": format!("vsock://2:{}/api", cfg.host_api.port), "vm_config": serde_json::to_string(&vm_config)?, }); + // No attestation trust anchor is ever written here. Simulated deployments + // receive only the development seed through `.tee-simulator.json`; the + // in-guest simulator derives the matching roots itself and publishes them + // to the guest verifier. A host cannot be allowed to choose the root that + // authenticates the guest's key provider. if let Some(mr_config) = mr_config { MrConfigV3::from_document(&mr_config).context("Invalid mr_config document")?; sys_config["mr_config"] = serde_json::to_value(mr_config)?; @@ -1429,13 +1830,16 @@ fn make_vm_config( requirements: Option<&dstack_types::Requirements>, ) -> Result<serde_json::Value> { let platform = cfg.cvm.resolved_platform(); - let is_amd_sev_snp = platform == crate::config::TeePlatform::AmdSevSnp && !manifest.no_tee; - let is_tdx = platform == crate::config::TeePlatform::Tdx && !manifest.no_tee; + let is_amd_sev_snp = platform == crate::config::CvmPlatform::AmdSevSnp && !manifest.no_tee; + let is_tdx = platform == crate::config::CvmPlatform::Tdx && !manifest.no_tee; + let is_gcp_tdx = manifest.simulated_tee == Some(dstack_types::TeeVariant::DstackGcpTdx); + let is_aws_nitro_tpm = + manifest.simulated_tee == Some(dstack_types::TeeVariant::DstackAwsNitroTpm); let tdx_attestation_variant = if is_tdx { tdx_attestation_variant_from_requirements(requirements).unwrap_or_else(|| { cfg.cvm .tdx_attestation_variant - .resolve(manifest.memory, image_supports_tdx_lite(image)) + .resolve(image_supports_tdx_lite(image)) }) } else { dstack_types::TdxAttestationVariant::Legacy @@ -1450,12 +1854,12 @@ fn make_vm_config( .and_then(|d| hex::decode(d).ok()) .unwrap_or_default(); // Attach the lite measurement material whenever the image provides it, - // regardless of the resolved attestation variant: the guest's exposed - // event log always retains the RTMR0 ACPI digest events (see - // cc_eventlog::tdx::label_tdx_acpi_data_events), so a verifier can freely - // choose lite verification for a legacy-resolved boot too. - // `tdx_attestation_variant` keeps its original meaning of "the scheme the - // VMM/KMS resolved for this boot" and is unaffected by this. + // regardless of the resolved attestation variant, so the config shape stays + // uniform across images. It does not widen how the boot is verified: + // verifiers select the path from `tdx_attestation_variant` alone, and a + // legacy-resolved boot is verified through the image download even with the + // document attached. `tdx_attestation_variant` keeps its original meaning of + // "the scheme the VMM/KMS resolved for this boot". let tdx_measurement = if is_tdx { if tdx_attestation_variant.is_lite() { Some(image.tdx_measurement.clone().context( @@ -1479,29 +1883,82 @@ fn make_vm_config( // ACPI/DSDT layout and therefore RTMR0. Measure the interface count so the // verifier reconstructs the exact device layout. let num_nics = resolved_networks(manifest, &cfg.cvm).len() as u32; + let num_verity_volumes = manifest.volumes.len() as u32; + let swtpm = manifest.swtpm; + let gcp_measurement = if is_gcp_tdx { + Some( + image + .gcp_measurement + .clone() + .context("GCP TDX image is missing measurement.gcp.cbor measurement material")?, + ) + } else { + None + }; + let aws_measurement = + if is_aws_nitro_tpm { + Some(image.aws_measurement.clone().context( + "AWS NitroTPM image is missing measurement.aws.cbor measurement material", + )?) + } else { + None + }; let mut config = serde_json::to_value(dstack_types::VmConfig { os_image_hash, cpu_count: effective_vcpus, memory_size: manifest.memory as u64 * 1024 * 1024, qemu_single_pass_add_pages: cfg.cvm.qemu_single_pass_add_pages, pic: cfg.cvm.qemu_pic, - qemu_version: cfg.cvm.qemu_version.clone(), + qemu_version: Some(cfg.cvm.resolve_qemu_version()?), pci_hole64_size: cfg.cvm.qemu_pci_hole64_size, hugepages: manifest.hugepages, num_gpus: gpus.gpus.len() as u32, num_nvswitches: gpus.bridges.len() as u32, num_nics, + num_verity_volumes, + swtpm, host_share_mode: cfg.cvm.host_share_mode.clone(), hotplug_off: cfg.cvm.qemu_hotplug_off, image: Some(manifest.image.clone()), ovmf_variant: image.info.ovmf_variant, tdx_attestation_variant, tdx_measurement, - gcp_measurement: None, - aws_measurement: None, + gcp_measurement, + aws_measurement, })?; // For backward compatibility config["spec_version"] = serde_json::Value::from(1); + if is_aws_nitro_tpm { + let replay = image + .aws_pcr_replay + .as_ref() + .context("AWS NitroTPM simulation requires measurement.aws.replay.json")?; + let replay_measurement = dstack_types::AwsOsImageMeasurement::from_boot_pcrs( + &replay.pcr4, + &replay.pcr7, + &replay.pcr12, + ) + .map_err(anyhow::Error::msg)?; + let image_measurement = image + .aws_measurement + .as_ref() + .context("AWS NitroTPM image is missing measurement.aws.cbor")? + .decode_measurement() + .map_err(anyhow::Error::msg)?; + anyhow::ensure!( + replay_measurement == image_measurement, + "measurement.aws.replay.json does not match measurement.aws.cbor" + ); + config["aws_pcr_replay"] = serde_json::to_value(replay)?; + } + if is_gcp_tdx { + config["gcp_tpm_replay"] = serde_json::to_value( + image + .gcp_tpm_replay + .as_ref() + .context("GCP TDX simulation requires measurement.gcp.eventlog.bin")?, + )?; + } if is_amd_sev_snp { if let Some(mr_config) = mr_config { MrConfigV3::from_document(&mr_config).context("Invalid mr_config document")?; @@ -1525,11 +1982,47 @@ fn make_vm_config( Ok(config) } +pub(crate) fn needs_swtpm( + key_provider: dstack_types::KeyProviderKind, + simulated_tee: Option<dstack_types::TeeVariant>, +) -> bool { + matches!(key_provider, dstack_types::KeyProviderKind::Tpm) + && !simulated_tee.is_some_and(mock_attestation::platform_provides_tpm) +} + #[cfg(test)] mod tests { + use super::mr_config::{mr_config_version, MrConfigVersion}; use super::*; + + #[test] + fn accepts_server_generated_ids() { + validate_vm_id(&uuid::Uuid::new_v4().to_string()).unwrap(); + validate_vm_id("3f2504e0-4f89-41d3-9a0c-0305e82c3301").unwrap(); + validate_vm_id("vm_1-A").unwrap(); + } + + #[test] + fn rejects_ids_that_would_escape_run_path() { + for id in [ + "", + "..", + "../../etc", + "/etc/passwd", + "a/b", + "a\\b", + "vm id", + ".", + "\0", + "café", + ] { + assert!(validate_vm_id(id).is_err(), "should reject {id:?}"); + } + assert!(validate_vm_id(&"a".repeat(65)).is_err()); + } + use crate::config::{ - load_config_figment, Networking, NetworkingMode, TdxAttestationVariantConfig, TeePlatform, + load_config_figment, CvmPlatform, NetworkingMode, TdxAttestationVariantConfig, }; use dstack_types::{ TdxImageMeasurement, TdxMrtdCandidates, TdxOsImageMeasurement, @@ -1538,9 +2031,301 @@ mod tests { use rocket::figment::Figment; use std::time::UNIX_EPOCH; + fn reserve(running: &[(&str, u32)], in_memory: &[(&str, u32)]) -> Vec<(u32, String)> { + let running: HashMap<String, u32> = running + .iter() + .map(|(id, cid)| (id.to_string(), *cid)) + .collect(); + cids_to_reserve( + &running, + in_memory.iter().map(|(id, cid)| (id.to_string(), *cid)), + ) + .into_iter() + .collect() + } + + #[test] + fn stopped_vms_keep_their_cid_reserved_across_a_reload() { + // "running" comes from the supervisor, so a stopped VM appears only in + // memory. Reserving just the running set would free 1001 and let the + // next allocate() hand it to a new VM. + let reserved = reserve( + &[("running-vm", 1000)], + &[("running-vm", 1000), ("stopped-vm", 1001)], + ); + assert_eq!( + reserved, + vec![ + (1000, "running-vm".to_string()), + (1001, "stopped-vm".to_string()), + ] + ); + } + + #[test] + fn a_vm_both_running_and_in_memory_is_reserved_once() { + let reserved = reserve(&[("vm-a", 1000)], &[("vm-a", 1000)]); + assert_eq!(reserved, vec![(1000, "vm-a".to_string())]); + } + + #[test] + fn running_state_wins_when_memory_disagrees_about_the_owner() { + // The supervisor knows who actually holds the CID right now. + let reserved = reserve(&[("live-owner", 1000)], &[("stale-owner", 1000)]); + assert_eq!(reserved, vec![(1000, "live-owner".to_string())]); + } + + #[test] + fn nothing_is_reserved_when_no_vm_holds_a_cid() { + assert!(reserve(&[], &[]).is_empty()); + } + fn hex_of(byte: u8, len: usize) -> String { hex::encode(vec![byte; len]) } + fn restart_config() -> crate::config::AutoRestartConfig { + crate::config::AutoRestartConfig { + enabled: true, + interval: 1, + max_retries: 3, + initial_backoff: 2, + max_backoff: 5, + reset_window: 10, + } + } + + #[test] + fn serial_log_is_rotatable_only_when_the_annotation_confirms_it() { + // A VM launched by the current binary. + let current = serde_json::to_string(&ProcessAnnotation { + kind: "cvm".into(), + live_for: None, + serial_logappend: true, + }) + .unwrap(); + assert!(serial_log_is_rotatable(¤t)); + + // A VM inherited from a VMM that predates the option: its QEMU holds + // the log without O_APPEND, so truncating it would punch a sparse hole + // and the file would spring straight back over the cap. + assert!(!serial_log_is_rotatable(r#"{"kind":"cvm"}"#)); + assert!(!serial_log_is_rotatable( + r#"{"kind":"cvm","live_for":null}"# + )); + + // Anything we cannot read must answer conservatively. + assert!(!serial_log_is_rotatable("")); + assert!(!serial_log_is_rotatable("not json")); + assert!(!serial_log_is_rotatable(r#"{"serial_logappend":"yes"}"#)); + } + + #[test] + fn cvm_annotation_marks_the_serial_log_rotatable() { + // The flag must survive the round trip the supervisor performs, and + // must not disturb how existing consumers classify the process. + let note = serde_json::to_string(&ProcessAnnotation { + kind: "cvm".into(), + live_for: None, + serial_logappend: true, + }) + .unwrap(); + let parsed: ProcessAnnotation = serde_json::from_str(¬e).unwrap(); + assert!(parsed.serial_logappend); + assert!(parsed.is_cvm()); + } + + #[test] + fn rotatable_logs_always_include_supervisor_written_logs() -> Result<()> { + let temp = tempfile::tempdir()?; + let workdir = VmWorkDir::new(temp.path()); + + // A VM inherited from an older VMM: QEMU holds serial.log without + // O_APPEND, so rotating it would punch a sparse hole. stdout and stderr + // are the supervisor's, always opened with append(true), so they stay + // eligible and keep their cap across a VMM upgrade. + let inherited = rotatable_logs(&workdir, false); + assert_eq!( + inherited, + vec![workdir.stdout_file(), workdir.stderr_file()] + ); + + let launched = rotatable_logs(&workdir, true); + assert_eq!( + launched, + vec![ + workdir.stdout_file(), + workdir.stderr_file(), + workdir.serial_file() + ] + ); + Ok(()) + } + + #[test] + fn log_retention_defaults() -> Result<()> { + // These come from the shipped vmm.toml, not from serde defaults, so + // this also pins that the values there parse into what they read as. + let config = test_tdx_config()?; + assert_eq!(config.cvm.log.max_bytes, 4 * 1024 * 1024); + assert_eq!(config.cvm.log.max_backups, 3); + assert_eq!(config.cvm.log.check_interval_secs, 5); + Ok(()) + } + + #[test] + fn auto_restart_policy_backs_off_caps_and_exhausts_once() { + let config = restart_config(); + let start = std::time::Instant::now(); + let mut state = AutoRestartState::default(); + assert_eq!( + state.observe_exited(start, &config), + AutoRestartDecision::Scheduled { delay_secs: 2 } + ); + assert_eq!( + state.observe_exited(start + std::time::Duration::from_secs(1), &config), + AutoRestartDecision::Wait + ); + assert_eq!( + state.observe_exited(start + std::time::Duration::from_secs(2), &config), + AutoRestartDecision::Restart { + attempt: 1, + next_delay_secs: 4 + } + ); + assert_eq!( + state.observe_exited(start + std::time::Duration::from_secs(6), &config), + AutoRestartDecision::Restart { + attempt: 2, + next_delay_secs: 5 + } + ); + assert_eq!( + state.observe_exited(start + std::time::Duration::from_secs(11), &config), + AutoRestartDecision::Restart { + attempt: 3, + next_delay_secs: 5 + } + ); + assert_eq!( + state.observe_exited(start + std::time::Duration::from_secs(12), &config), + AutoRestartDecision::Exhausted { attempts: 3 } + ); + assert_eq!( + state.observe_exited(start + std::time::Duration::from_secs(20), &config), + AutoRestartDecision::Wait + ); + } + + #[test] + fn auto_restart_policy_resets_only_after_healthy_window() { + let config = restart_config(); + let start = std::time::Instant::now(); + let mut state = AutoRestartState::default(); + state.observe_exited(start, &config); + state.observe_exited(start + std::time::Duration::from_secs(2), &config); + assert!(!state.observe_running(start + std::time::Duration::from_secs(3), 10)); + assert!(!state.observe_running(start + std::time::Duration::from_secs(12), 10)); + assert!(state.observe_running(start + std::time::Duration::from_secs(13), 10)); + assert_eq!(state.attempts, 0); + assert_eq!( + state.observe_exited(start + std::time::Duration::from_secs(14), &config), + AutoRestartDecision::Scheduled { delay_secs: 2 } + ); + } + + #[test] + fn simulator_config_is_written_separately_with_measurement_inputs() -> Result<()> { + let dir = tempfile::tempdir()?; + let config = dstack_types::TeeSimulatorConfig { + platform: dstack_types::TeeVariant::DstackAmdSevSnp, + mock_attestation_seed: Some("11".repeat(32)), + collateral_base_url: Some("http://127.0.0.1:18088".into()), + ..Default::default() + }; + let mr_config = r#"{"version":3}"#; + let vm_config = format!( + r#"{{"image":"dev","aws_pcr_replay":{{"version":1,"events":[],"pcr4":"{zero}","pcr7":"{zero}","pcr12":"{zero}"}},"gcp_tpm_replay":{{"event_log":"AQID"}}}}"#, + zero = "00".repeat(48) + ); + let sys_config = serde_json::json!({ + "kms_urls": [], + "gateway_urls": [], + "vm_config": vm_config, + "mr_config": mr_config, + }) + .to_string(); + + sync_tee_simulator_config(dir.path(), Some(&config), &sys_config)?; + + let written: dstack_types::TeeSimulatorConfig = + serde_json::from_slice(&fs::read(dir.path().join(TEE_SIMULATOR_CONFIG))?)?; + assert_eq!(written.platform, config.platform); + assert_eq!(written.mock_attestation_seed, config.mock_attestation_seed); + assert_eq!(written.collateral_base_url, config.collateral_base_url); + assert_eq!(written.mr_config.as_deref(), Some(mr_config)); + assert_eq!(written.vm_config.as_deref(), Some(vm_config.as_str())); + assert_eq!( + written.aws_pcr_replay.as_ref().map(|replay| replay.version), + Some(1) + ); + assert_eq!( + written + .gcp_tpm_replay + .as_ref() + .map(|replay| replay.event_log.as_slice()), + Some([1, 2, 3].as_slice()) + ); + + sync_tee_simulator_config(dir.path(), None, &sys_config)?; + assert!(!dir.path().join(TEE_SIMULATOR_CONFIG).exists()); + Ok(()) + } + + #[test] + fn instance_platform_overrides_node_simulator_template() -> Result<()> { + let mut config = test_tdx_config()?; + config.cvm.tee_simulator = Some(dstack_types::TeeSimulatorConfig { + platform: dstack_types::TeeVariant::DstackTdx, + mock_attestation_seed: Some("11".repeat(32)), + ..Default::default() + }); + let mut manifest = test_manifest(2048); + assert!(simulator_config_for_manifest(&config.cvm, &manifest)?.is_none()); + manifest.simulated_tee = Some(dstack_types::TeeVariant::DstackNitroEnclave); + + let resolved = simulator_config_for_manifest(&config.cvm, &manifest)? + .context("simulator config should be enabled")?; + + assert_eq!( + resolved.platform, + dstack_types::TeeVariant::DstackNitroEnclave + ); + Ok(()) + } + + #[test] + fn gpu_config_has_gpus_only_when_resolved_gpu_list_is_non_empty() { + assert!(!GpuConfig::default().has_gpus()); + assert!(!GpuConfig { + attach_mode: AttachMode::All, + ..Default::default() + } + .has_gpus()); + assert!(!GpuConfig { + bridges: vec![GpuSpec { + slot: "0000:01:00.0".into(), + }], + ..Default::default() + } + .has_gpus()); + assert!(GpuConfig { + gpus: vec![GpuSpec { + slot: "0000:02:00.0".into(), + }], + ..Default::default() + } + .has_gpus()); + } #[test] fn put_manifest_keeps_legacy_networking_for_rollback() -> Result<()> { @@ -1550,15 +2335,10 @@ mod tests { )); let workdir = VmWorkDir::new(&temp); let mut manifest = test_manifest(1024); - manifest.networks = vec![Networking { + manifest.networks = vec![NicNetworking { mode: NetworkingMode::Bridge, bridge: "dstack-br0".to_string(), - forward_service_enabled: true, - mac_prefix: String::new(), - net: String::new(), - dhcp_start: String::new(), - restrict: false, - netdev: String::new(), + ..NicNetworking::default() }]; workdir.put_manifest(&manifest)?; @@ -1681,14 +2461,59 @@ mod tests { kms_urls: vec![], gateway_urls: vec![], no_tee: false, + simulated_tee: None, + swtpm: false, networks: vec![], + volumes: vec![], } } + #[test] + fn selects_mr_config_version_for_each_tee_mode() -> Result<()> { + let manifest = test_manifest(2048); + assert_eq!( + mr_config_version(&manifest, CvmPlatform::AmdSevSnp, false, false)?, + Some(MrConfigVersion::V3) + ); + assert_eq!( + mr_config_version(&manifest, CvmPlatform::Tdx, false, false)?, + None + ); + assert_eq!( + mr_config_version(&manifest, CvmPlatform::Tdx, true, false)?, + Some(MrConfigVersion::V1) + ); + assert_eq!( + mr_config_version(&manifest, CvmPlatform::Tdx, true, true)?, + Some(MrConfigVersion::V3) + ); + assert_eq!( + mr_config_version(&manifest, CvmPlatform::Tdx, false, true) + .err() + .map(|error| error.to_string()), + Some("key provider ID requires MrConfigV3, but use_mrconfigid is disabled".to_string()) + ); + + let mut no_tee = manifest.clone(); + no_tee.no_tee = true; + assert_eq!( + mr_config_version(&no_tee, CvmPlatform::AmdSevSnp, true, true)?, + None + ); + + no_tee.simulated_tee = Some(dstack_types::TeeVariant::DstackAmdSevSnp); + assert_eq!( + mr_config_version(&no_tee, CvmPlatform::Tdx, false, false)?, + Some(MrConfigVersion::V3) + ); + Ok(()) + } + fn dummy_tdx_measurement_document() -> TdxOsImageMeasurementDocument { let measurement = TdxOsImageMeasurement { + kernel_header_normalized: true, image: TdxImageMeasurement { - kernel_cmdline_sha384: vec![0x10; 48], + base_cmdline: "console=ttyS0 dstack.rootfs_hash=10".to_string(), kernel_authenticode: vec![0x20; 48], initrd_sha384: vec![0x30; 48], }, @@ -1737,13 +2562,20 @@ mod tests { digest: Some(hex_of(0xaa, 32)), tdx_measurement, sev_measurement: None, + gcp_measurement: None, + aws_measurement: None, + aws_pcr_replay: None, + gcp_tpm_replay: None, } } fn test_tdx_config() -> Result<Config> { let mut config: Config = Figment::from(load_config_figment(None)).extract()?; - config.cvm.platform = Some(TeePlatform::Tdx); + config.cvm.platform = Some(CvmPlatform::Tdx); config.cvm.tdx_attestation_variant = TdxAttestationVariantConfig::Auto; + // No QEMU binary under test; declare the version the way a host + // without a detectable one has to. + config.cvm.qemu_version = Some("9.2.1".to_string()); Ok(config) } @@ -1765,15 +2597,10 @@ mod tests { fn vm_measurement_config_ignores_networking_changes() -> Result<()> { let config = test_tdx_config()?; let mut bridge_manifest = test_manifest(2048); - bridge_manifest.networks = vec![Networking { + bridge_manifest.networks = vec![NicNetworking { mode: NetworkingMode::Bridge, bridge: "dstack-br0".to_string(), - forward_service_enabled: true, - mac_prefix: "02:aa:bb".to_string(), - net: String::new(), - dhcp_start: String::new(), - restrict: false, - netdev: String::new(), + ..NicNetworking::default() }]; let user_manifest = test_manifest(2048); let image = test_tdx_image(true); @@ -1789,16 +2616,60 @@ mod tests { } #[test] - fn tdx_auto_variant_uses_legacy_for_low_non_2g_memory() -> Result<()> { + fn vm_measurement_config_includes_verity_volume_count() -> Result<()> { + let config = test_tdx_config()?; + let mut manifest = test_manifest(2048); + manifest.volumes = vec![ + VmVolume { + source: "/volumes/a.img".into(), + }, + VmVolume { + source: "/volumes/b.img".into(), + }, + ]; + let vm_config = make_vm_config( + &config, + &manifest, + &test_tdx_image(true), + &hex_of(0x22, 32), + None, + None, + )?; + + assert_eq!(vm_config["num_verity_volumes"], 2); + Ok(()) + } + + #[test] + fn vm_measurement_config_includes_swtpm() -> Result<()> { + let config = test_tdx_config()?; + let mut manifest = test_manifest(2048); + manifest.swtpm = true; + + let vm_config = make_vm_config( + &config, + &manifest, + &test_tdx_image(true), + &hex_of(0x22, 32), + None, + None, + )?; + + assert_eq!(vm_config["swtpm"], true); + Ok(()) + } + + /// 1 GiB used to fall back to legacy: it is below 3 GiB and not the 2 GiB + /// exemption. That heuristic existed for images whose OVMF leaves the setup + /// header for QEMU to rewrite, which the build system no longer produces. + #[test] + fn tdx_auto_variant_uses_lite_for_low_non_2g_memory() -> Result<()> { let config = test_tdx_config()?; let manifest = test_manifest(1024); let image = test_tdx_image(true); let vm_config = make_vm_config(&config, &manifest, &image, &hex_of(0x22, 32), None, None)?; - assert!(vm_config.get("tdx_attestation_variant").is_none()); - // tdx_measurement is attached whenever the image supports it, even - // when the resolved variant is legacy, so a verifier can still - // choose lite verification for this boot. + assert_eq!(vm_config["tdx_attestation_variant"], "lite"); assert!(vm_config.get("tdx_measurement").is_some()); assert_eq!( vm_config["os_image_hash"] @@ -1924,7 +2795,9 @@ mod tests { let mut config: Config = Figment::from(load_config_figment(None)).extract()?; config.image.path = image_root; - config.cvm.platform = Some(TeePlatform::AmdSevSnp); + config.cvm.platform = Some(CvmPlatform::AmdSevSnp); + config.cvm.qemu_version = Some("9.2.1".to_string()); + config.cvm.nvidia_attestation_proxy_url = Some("http://10.0.2.2:8090".to_string()); let compose_hash = hex_of(0x22, 32); let manifest = Manifest { id: "snp-test".to_string(), @@ -1942,12 +2815,16 @@ mod tests { kms_urls: vec![], gateway_urls: vec![], no_tee: false, + simulated_tee: None, + swtpm: false, networks: vec![], + volumes: vec![], }; let mr_config = MrConfigV3::new( vec![0x11; 20], vec![0x22; 32], + None, dstack_types::KeyProviderKind::None, vec![], vec![0x44; 20], @@ -1982,6 +2859,22 @@ mod tests { let sys_config_document = make_sys_config(&config, &manifest, &compose_hash, Some(mr_config), None)?; let sys_config: serde_json::Value = serde_json::from_str(&sys_config_document)?; + assert!(sys_config.get("tee_simulator").is_none()); + // A host must never nominate the trust anchor that authenticates its + // guest's key provider, in any deployment mode. Simulated guests derive + // their own roots from the seed in `.tee-simulator.json` instead. + for key in sys_config + .as_object() + .context("sys-config must be an object")? + .keys() + { + assert!( + !key.contains("root_ca") && !key.contains("trust_anchor"), + "sys-config must not carry an attestation trust anchor: {key}" + ); + } + assert_eq!(sys_config["pccs_url"], config.cvm.pccs_url); + assert_eq!(sys_config["collateral_urls"]["pccs"], config.cvm.pccs_url); let vm_config: serde_json::Value = serde_json::from_str( sys_config["vm_config"] .as_str() @@ -2000,8 +2893,13 @@ mod tests { .context("mr_config must be a string")?; let parsed_mr_config = MrConfigV3::from_document(mr_config_document)?; - assert_eq!(parsed_mr_config.app_id, vec![0x11; 20]); + assert_eq!( + sys_config["nvidia_attestation_proxy_url"], + "http://10.0.2.2:8090" + ); + assert_eq!(parsed_mr_config.app_id, Some(vec![0x11; 20])); assert_eq!(parsed_mr_config.compose_hash, vec![0x22; 32]); + assert_eq!(parsed_mr_config.gpu_policy_hash, None); assert_eq!(vm_config["mr_config"], sys_config["mr_config"]); assert_eq!( vm_config["os_image_hash"] @@ -2043,6 +2941,24 @@ mod tests { } } +/// CIDs that must survive a pool rebuild, mapped to the VM that owns each one. +/// +/// Running processes are the authoritative source for CIDs currently on the +/// wire, but they are not the whole picture: a stopped VM still records its CID +/// in `vm.config.cid` and keeps it on restart, so it has to stay reserved too. +/// Reserving only the running set would let `allocate()` hand the same CID to a +/// new VM, and the two would collide the moment the stopped one starts. +/// +/// Keyed by CID so a VM that is both running and in memory is reserved once. +fn cids_to_reserve( + running: &HashMap<String, u32>, + in_memory: impl Iterator<Item = (String, u32)>, +) -> BTreeMap<u32, String> { + let mut reserved: BTreeMap<u32, String> = in_memory.map(|(id, cid)| (cid, id)).collect(); + reserved.extend(running.iter().map(|(id, cid)| (*cid, id.clone()))); + reserved +} + fn paginate<T>(items: Vec<T>, page: u32, page_size: u32) -> impl Iterator<Item = T> { let skip; let take; @@ -2064,15 +2980,86 @@ pub struct VmState { state: VmStateMut, } +/// Per-process retry bookkeeping; intentionally reset whenever the VMM restarts. +#[derive(Debug, Clone, Default)] +struct AutoRestartState { + attempts: u32, + next_retry: Option<std::time::Instant>, + healthy_since: Option<std::time::Instant>, + exhausted_reported: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AutoRestartDecision { + Wait, + Scheduled { delay_secs: u64 }, + Restart { attempt: u32, next_delay_secs: u64 }, + Exhausted { attempts: u32 }, +} + +impl AutoRestartState { + fn reset(&mut self) { + *self = Self::default(); + } + + fn observe_running(&mut self, now: std::time::Instant, reset_window: u64) -> bool { + let healthy_since = self.healthy_since.get_or_insert(now); + if self.attempts > 0 + && now.duration_since(*healthy_since) >= std::time::Duration::from_secs(reset_window) + { + self.reset(); + return true; + } + false + } + + fn observe_exited( + &mut self, + now: std::time::Instant, + config: &crate::config::AutoRestartConfig, + ) -> AutoRestartDecision { + self.healthy_since = None; + if self.attempts >= config.max_retries { + if self.exhausted_reported { + return AutoRestartDecision::Wait; + } + self.exhausted_reported = true; + return AutoRestartDecision::Exhausted { + attempts: self.attempts, + }; + } + let Some(next_retry) = self.next_retry else { + self.next_retry = Some(now + std::time::Duration::from_secs(config.initial_backoff)); + return AutoRestartDecision::Scheduled { + delay_secs: config.initial_backoff, + }; + }; + if now < next_retry { + return AutoRestartDecision::Wait; + } + self.attempts += 1; + let shift = self.attempts.min(63); + let delay_secs = config + .initial_backoff + .saturating_mul(1u64 << shift) + .min(config.max_backoff); + self.next_retry = Some(now + std::time::Duration::from_secs(delay_secs)); + AutoRestartDecision::Restart { + attempt: self.attempts, + next_delay_secs: delay_secs, + } + } +} + #[derive(Debug, Clone, Default)] struct VmStateMut { boot_progress: String, boot_error: String, shutdown_progress: String, - guest_ip: String, runtime_networks: Vec<Networking>, devices: GpuConfig, events: VecDeque<pb::GuestEvent>, + auto_restart: AutoRestartState, /// True when the VM is being removed (cleanup in progress). removing: bool, } @@ -2102,21 +3089,11 @@ impl VmState { state: VmStateMut::default(), } } - - pub fn effective_networks(&self, cvm: &crate::config::CvmConfig) -> Vec<Networking> { - if self.state.runtime_networks.is_empty() { - resolved_networks(&self.config.manifest, cvm) - } else { - self.state.runtime_networks.clone() - } - } } pub(crate) struct AppState { cid_pool: IdPool<u32>, vms: HashMap<String, VmState>, - /// Tracks active port forwarding rules per VM ID (bridge mode only). - active_forwards: HashMap<String, Vec<ForwardRule>>, } impl AppState { @@ -2140,3 +3117,19 @@ impl AppState { self.vms.values() } } + +/// Reject VM ids that would escape `run_path` once joined into a filesystem +/// path. Ids are server-generated UUIDs, so hex digits plus `-` covers every +/// legitimate value; `Path::join` replaces the base outright on an absolute +/// path, and `..` walks out of it, so neither may reach the join. +pub(crate) fn validate_vm_id(id: &str) -> Result<()> { + if id.is_empty() + || id.len() > 64 + || !id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') + { + bail!("invalid VM id"); + } + Ok(()) +} diff --git a/dstack/vmm/src/app/host_share.rs b/dstack/vmm/src/app/host_share.rs index df5b1b536..027837edb 100644 --- a/dstack/vmm/src/app/host_share.rs +++ b/dstack/vmm/src/app/host_share.rs @@ -5,12 +5,13 @@ //! Host-to-guest shared-file disk creation. use std::io::{Seek, SeekFrom, Write}; -use std::path::Path; +use std::{fs::Permissions, os::unix::fs::PermissionsExt, path::Path}; use anyhow::{Context, Result}; use dstack_types::shared_filenames::HOST_SHARED_DISK_LABEL; use fatfs::{FileSystem, FormatVolumeOptions, FsOptions}; use fs_err as fs; +use tempfile::NamedTempFile; /// Creates a FAT32 disk image containing the files in `shared_dir`. pub(super) fn create_shared_disk( @@ -19,6 +20,17 @@ pub(super) fn create_shared_disk( ) -> Result<()> { let disk_path = disk_path.as_ref(); let shared_dir = shared_dir.as_ref(); + let parent = disk_path.parent().unwrap_or_else(|| Path::new(".")); + let mut temporary = NamedTempFile::new_in(parent).with_context(|| { + format!( + "failed to create temporary disk image in {}", + parent.display() + ) + })?; + temporary + .as_file() + .set_permissions(Permissions::from_mode(0o644)) + .context("failed to set disk image permissions")?; // Must be large enough to hold all host-shared files (app-compose.json and // .user-config can each be up to 50 MiB, see HostShared::copy) plus FAT32 overhead. @@ -26,13 +38,7 @@ pub(super) fn create_shared_disk( // Back the image by a file (sparse until written) and stream files into it so // peak memory stays bounded regardless of DISK_SIZE or input file sizes. - let mut image = fs::OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(true) - .open(disk_path) - .with_context(|| format!("failed to create disk image at {}", disk_path.display()))?; + let image = temporary.as_file_mut(); image .set_len(DISK_SIZE) .context("failed to size disk image")?; @@ -45,20 +51,23 @@ pub(super) fn create_shared_disk( let format_opts = FormatVolumeOptions::new() .fat_type(fatfs::FatType::Fat32) .volume_label(label_bytes); - fatfs::format_volume(&mut image, format_opts).context("failed to format disk as FAT32")?; + fatfs::format_volume(&mut *image, format_opts).context("failed to format disk as FAT32")?; } image .seek(SeekFrom::Start(0)) .context("failed to seek to start")?; - let filesystem = - FileSystem::new(&mut image, FsOptions::new()).context("failed to open FAT32 filesystem")?; + let filesystem = FileSystem::new(&mut *image, FsOptions::new()) + .context("failed to open FAT32 filesystem")?; let root_dir = filesystem.root_dir(); for entry in fs::read_dir(shared_dir).context("failed to read shared directory")? { let entry = entry.context("failed to read directory entry")?; let path = entry.path(); - if !path.is_file() { + let file_type = entry + .file_type() + .context("failed to inspect shared entry")?; + if !file_type.is_file() { continue; } @@ -74,5 +83,116 @@ pub(super) fn create_shared_disk( destination.flush().context("failed to flush FAT32 file")?; } + drop(root_dir); + drop(filesystem); + temporary + .as_file() + .sync_all() + .context("failed to sync temporary disk image")?; + temporary + .persist(disk_path) + .map_err(|error| error.error) + .with_context(|| format!("failed to publish disk image at {}", disk_path.display()))?; + fs::File::open(parent) + .with_context(|| format!("failed to open disk image directory {}", parent.display()))? + .sync_all() + .with_context(|| format!("failed to sync disk image directory {}", parent.display()))?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::create_shared_disk; + use fatfs::{FileSystem, FsOptions}; + use std::{fs, io::Read, os::unix::fs::PermissionsExt, sync::Arc, thread}; + use tempfile::TempDir; + + fn read_file(disk: &std::path::Path, name: &str) -> Vec<u8> { + let mut image = fs::OpenOptions::new() + .read(true) + .write(true) + .open(disk) + .unwrap(); + let filesystem = FileSystem::new(&mut image, FsOptions::new()).unwrap(); + let mut file = filesystem.root_dir().open_file(name).unwrap(); + let mut contents = Vec::new(); + file.read_to_end(&mut contents).unwrap(); + contents + } + + #[test] + fn creates_fixed_size_disk_with_exact_regular_file_contents() { + let root = TempDir::new().unwrap(); + let shared = root.path().join("shared"); + fs::create_dir(&shared).unwrap(); + fs::write(shared.join("app-compose.json"), b"compose").unwrap(); + fs::create_dir(shared.join("ignored-directory")).unwrap(); + let disk = root.path().join("host-shared.img"); + create_shared_disk(&disk, &shared).unwrap(); + assert_eq!(fs::metadata(&disk).unwrap().len(), 128 * 1024 * 1024); + assert_eq!( + fs::metadata(&disk).unwrap().permissions().mode() & 0o777, + 0o644 + ); + assert_eq!(read_file(&disk, "app-compose.json"), b"compose"); + } + + #[test] + fn missing_or_oversized_sources_never_publish_partial_disk() { + let root = TempDir::new().unwrap(); + let missing_disk = root.path().join("missing.img"); + assert!(create_shared_disk(&missing_disk, root.path().join("absent")).is_err()); + assert!(!missing_disk.exists()); + + let shared = root.path().join("oversized"); + fs::create_dir(&shared).unwrap(); + let source = fs::File::create(shared.join("too-large")).unwrap(); + source.set_len(129 * 1024 * 1024).unwrap(); + let oversized_disk = root.path().join("oversized.img"); + assert!(create_shared_disk(&oversized_disk, &shared).is_err()); + assert!(!oversized_disk.exists()); + } + + #[cfg(unix)] + #[test] + fn symlinked_sources_cannot_escape_shared_root() { + use std::os::unix::fs::symlink; + let root = TempDir::new().unwrap(); + let shared = root.path().join("shared"); + fs::create_dir(&shared).unwrap(); + fs::write(root.path().join("outside-secret"), b"must-not-copy").unwrap(); + fs::write(shared.join("regular"), b"regular").unwrap(); + symlink(root.path().join("outside-secret"), shared.join("escape")).unwrap(); + let disk = root.path().join("host-shared.img"); + create_shared_disk(&disk, &shared).unwrap(); + assert_eq!(read_file(&disk, "regular"), b"regular"); + let mut image = fs::OpenOptions::new() + .read(true) + .write(true) + .open(disk) + .unwrap(); + let filesystem = FileSystem::new(&mut image, FsOptions::new()).unwrap(); + assert!(filesystem.root_dir().open_file("escape").is_err()); + } + + #[test] + fn concurrent_publication_never_exposes_partial_image() { + let root = TempDir::new().unwrap(); + let shared = root.path().join("shared"); + fs::create_dir(&shared).unwrap(); + fs::write(shared.join("payload"), vec![7_u8; 1024 * 1024]).unwrap(); + let disk = Arc::new(root.path().join("host-shared.img")); + let shared = Arc::new(shared); + let workers: Vec<_> = (0..2) + .map(|_| { + let disk = Arc::clone(&disk); + let shared = Arc::clone(&shared); + thread::spawn(move || create_shared_disk(disk.as_path(), shared.as_path())) + }) + .collect(); + for worker in workers { + worker.join().unwrap().unwrap(); + } + assert_eq!(read_file(&disk, "payload"), vec![7_u8; 1024 * 1024]); + } +} diff --git a/dstack/vmm/src/app/id_pool.rs b/dstack/vmm/src/app/id_pool.rs index 5bbb205e0..f11b790ff 100644 --- a/dstack/vmm/src/app/id_pool.rs +++ b/dstack/vmm/src/app/id_pool.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use std::collections::BTreeSet; +use std::fmt::Display; use anyhow::bail; @@ -15,12 +16,15 @@ macro_rules! impl_numbers { })* }; } -pub trait Number: Ord + Sized + Clone { +pub trait Number: Ord + Sized + Clone + Display { fn next(&self) -> Option<Self>; } impl_numbers!(u8, u16, u32, u64, u128); impl_numbers!(i8, i16, i32, i64, i128); +/// A pool of ids over the half-open interval `[start, end)`. +/// +/// `start >= end` is a valid but permanently empty pool. pub struct IdPool<T: Ord = u32> { start: T, end: T, @@ -36,27 +40,34 @@ impl<T: Number> IdPool<T> { } } + /// Mark `id` as in use. Fails if `id` falls outside `[start, end)` or is + /// already taken. + /// + /// Callers reconstructing state from ids this pool did not hand out (e.g. + /// CIDs of processes started under an earlier configuration) should treat + /// the out-of-range error as a warning rather than a hard failure. pub fn occupy(&mut self, id: T) -> anyhow::Result<()> { - if self.allocated.insert(id) { - Ok(()) - } else { - bail!("id already occupied") + if id < self.start || id >= self.end { + bail!("id {id} outside pool range [{}, {})", self.start, self.end); } + if !self.allocated.insert(id.clone()) { + bail!("id {id} already occupied"); + } + Ok(()) } pub fn allocate(&mut self) -> Option<T> { let mut id = self.start.clone(); - while let Some(next) = id.next() { - if next >= self.end { + loop { + if id >= self.end { return None; } - if !self.allocated.contains(&next) { - self.allocated.insert(next.clone()); - return Some(next); + if !self.allocated.contains(&id) { + self.allocated.insert(id.clone()); + return Some(id); } - id = next; + id = id.next()?; } - None } pub fn free(&mut self, id: T) { @@ -67,3 +78,78 @@ impl<T: Number> IdPool<T> { self.allocated.clear(); } } + +#[cfg(test)] +mod tests { + use super::IdPool; + + #[test] + fn allocation_is_start_inclusive_and_end_exclusive() { + let mut pool = IdPool::new(10_u8, 13); + assert_eq!(pool.allocate(), Some(10)); + assert_eq!(pool.allocate(), Some(11)); + assert_eq!(pool.allocate(), Some(12)); + assert_eq!(pool.allocate(), None); + } + + #[test] + fn occupied_ids_are_unique_range_bounded_and_reusable() { + let mut pool = IdPool::new(10_u8, 13); + assert!(pool.occupy(9).is_err()); + assert!(pool.occupy(13).is_err()); + pool.occupy(10).unwrap(); + assert!(pool.occupy(10).is_err()); + assert_eq!(pool.allocate(), Some(11)); + pool.free(10); + assert_eq!(pool.allocate(), Some(10)); + pool.clear(); + assert_eq!(pool.allocate(), Some(10)); + } + + #[test] + fn repeated_allocation_never_hands_out_a_duplicate() { + let mut pool = IdPool::new(20_u8, 28); + let mut allocated: Vec<_> = std::iter::from_fn(|| pool.allocate()).collect(); + assert_eq!(pool.allocate(), None); + allocated.sort_unstable(); + assert_eq!(allocated, (20_u8..28).collect::<Vec<_>>()); + } + + #[test] + fn reconstruction_from_occupied_ids_preserves_uniqueness() { + // mirrors reload_vms(): rebuild the pool from the ids of processes + // already running, then keep allocating from what is left. + let mut pool = IdPool::new(20_u8, 28); + for id in 20_u8..28 { + pool.occupy(id).unwrap(); + } + assert_eq!(pool.allocate(), None); + assert!(pool.occupy(20).is_err()); + pool.free(23); + assert_eq!(pool.allocate(), Some(23)); + } + + #[test] + fn exhaustion_at_the_numeric_maximum_returns_none() { + // end == T::MAX, so the last allocatable id is MAX - 1 and the walk + // stops on the range check before `next()` can overflow. + let mut pool = IdPool::new(u8::MAX - 1, u8::MAX); + assert_eq!(pool.allocate(), Some(u8::MAX - 1)); + assert_eq!(pool.allocate(), None); + assert!(pool.occupy(u8::MAX).is_err()); + } + + #[test] + fn an_empty_range_allocates_nothing() { + let mut pool = IdPool::new(u8::MAX, u8::MAX); + assert_eq!(pool.allocate(), None); + assert!(pool.occupy(u8::MAX).is_err()); + } + + #[test] + fn out_of_range_error_names_the_id_and_the_bounds() { + let mut pool = IdPool::new(10_u8, 13); + let err = pool.occupy(13).unwrap_err().to_string(); + assert_eq!(err, "id 13 outside pool range [10, 13)"); + } +} diff --git a/dstack/vmm/src/app/image.rs b/dstack/vmm/src/app/image.rs index 40f7df4fd..fcb5600f6 100644 --- a/dstack/vmm/src/app/image.rs +++ b/dstack/vmm/src/app/image.rs @@ -8,11 +8,16 @@ use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; use dstack_types::{ - SevOsImageMeasurementDocument, TdxOsImageMeasurementDocument, SNP_MEASUREMENT_FILENAME, - TDX_MEASUREMENT_FILENAME, + version::Version, AwsOsImageMeasurementDocument, AwsPcrReplay, GcpOsImageMeasurementDocument, + GcpTpmReplay, SevOsImageMeasurementDocument, TdxOsImageMeasurementDocument, + GCP_MEASUREMENT_FILENAME, SNP_MEASUREMENT_FILENAME, TDX_MEASUREMENT_FILENAME, }; use serde::{Deserialize, Serialize}; +const AWS_MEASUREMENT_FILENAME: &str = "measurement.aws.cbor"; +const AWS_PCR_REPLAY_FILENAME: &str = "measurement.aws.replay.json"; +const GCP_TPM_EVENT_LOG_FILENAME: &str = "measurement.gcp.eventlog.bin"; + #[derive(Debug, Serialize, Deserialize)] pub struct ImageInfo { pub cmdline: Option<String>, @@ -41,18 +46,17 @@ pub struct ImageInfo { } impl ImageInfo { - pub fn version_tuple(&self) -> Option<(u16, u16, u16)> { - let version = self - .version - .split('.') - .take(3) - .map(|v| v.parse::<u16>()) - .collect::<Result<Vec<_>, _>>() - .ok()?; - if version.len() < 3 { - return None; - } - Some((version[0], version[1], version[2])) + /// Parse `version` with the shared dstack version grammar. + /// + /// Prerelease and build-metadata suffixes are stripped, so `0.6.1-rc1`, + /// `0.6.1.rc1` and `0.6.1` all compare equal. That is deliberate: image + /// version comparisons here gate guest features, and a release candidate + /// is built from the code of the version it is a candidate for. + /// + /// Returns `None` when `version` is absent or unparseable; callers must + /// decide explicitly whether that is fatal. + pub fn version(&self) -> Option<Version> { + Version::parse(&self.version) } } @@ -79,6 +83,14 @@ pub struct Image { pub tdx_measurement: Option<TdxOsImageMeasurementDocument>, /// AMD SEV-SNP no-image-download measurement material. pub sev_measurement: Option<SevOsImageMeasurementDocument>, + /// GCP TDX no-image-download measurement material. + pub gcp_measurement: Option<GcpOsImageMeasurementDocument>, + /// AWS NitroTPM no-image-download measurement material. + pub aws_measurement: Option<AwsOsImageMeasurementDocument>, + /// AWS boot events consumed only by the development NitroTPM simulator. + pub aws_pcr_replay: Option<AwsPcrReplay>, + /// GCP TPM event log consumed only by the development simulator. + pub gcp_tpm_replay: Option<GcpTpmReplay>, } impl Image { @@ -96,14 +108,18 @@ impl Image { impl Image { pub fn load(base_path: impl AsRef<Path>) -> Result<Self> { - let base_path = base_path.as_ref().absolutize()?; + let base_path = base_path + .as_ref() + .absolutize()? + .canonicalize() + .context("failed to resolve image directory")?; let mut info = ImageInfo::load(base_path.join("metadata.json"))?; - let initrd = base_path.join(&info.initrd); - let kernel = base_path.join(&info.kernel); - let hda = info.hda.as_ref().map(|hda| base_path.join(hda)); - let rootfs = info.rootfs.as_ref().map(|rootfs| base_path.join(rootfs)); - let bios = info.bios.as_ref().map(|bios| base_path.join(bios)); - let bios_sev = info.bios_sev.as_ref().map(|bios| base_path.join(bios)); + let initrd = resolve_artifact(&base_path, &info.initrd, "Initrd")?; + let kernel = resolve_artifact(&base_path, &info.kernel, "Kernel")?; + let hda = resolve_optional_artifact(&base_path, info.hda.as_deref(), "Hda")?; + let rootfs = resolve_optional_artifact(&base_path, info.rootfs.as_deref(), "Rootfs")?; + let bios = resolve_optional_artifact(&base_path, info.bios.as_deref(), "Bios")?; + let bios_sev = resolve_optional_artifact(&base_path, info.bios_sev.as_deref(), "SEV bios")?; let digest = fs::read_to_string(base_path.join("digest.txt")) .ok() .map(|s| s.trim().to_string()); @@ -148,6 +164,38 @@ impl Image { )), _ => None, }; + let gcp_measurement = load_measurement_document( + &base_path, + &sha256sum, + GCP_MEASUREMENT_FILENAME, + GcpOsImageMeasurementDocument::new, + )?; + let aws_measurement = load_measurement_document( + &base_path, + &sha256sum, + AWS_MEASUREMENT_FILENAME, + AwsOsImageMeasurementDocument::new, + )?; + let aws_pcr_replay_path = base_path.join(AWS_PCR_REPLAY_FILENAME); + let aws_pcr_replay = if aws_pcr_replay_path.exists() { + Some( + serde_json::from_slice(&fs::read(&aws_pcr_replay_path).with_context(|| { + format!("failed to read {}", aws_pcr_replay_path.display()) + })?) + .with_context(|| format!("failed to parse {}", aws_pcr_replay_path.display()))?, + ) + } else { + None + }; + let gcp_event_log_path = base_path.join(GCP_TPM_EVENT_LOG_FILENAME); + let gcp_tpm_replay = if gcp_event_log_path.exists() { + Some(GcpTpmReplay { + event_log: fs::read(&gcp_event_log_path) + .with_context(|| format!("failed to read {}", gcp_event_log_path.display()))?, + }) + } else { + None + }; if info.version.is_empty() { // Older images does not have version field. Fallback to the version of the image folder name info.version = guess_version(&base_path).unwrap_or_default(); @@ -163,6 +211,10 @@ impl Image { digest, tdx_measurement, sev_measurement, + gcp_measurement, + aws_measurement, + aws_pcr_replay, + gcp_tpm_replay, } .ensure_exists() } @@ -198,6 +250,48 @@ impl Image { } } +fn resolve_artifact(base_path: &Path, value: &str, label: &str) -> Result<PathBuf> { + let candidate = base_path.join(value); + let resolved = candidate + .canonicalize() + .with_context(|| format!("{label} does not exist: {}", candidate.display()))?; + if !resolved.starts_with(base_path) { + bail!("{label} escapes image directory: {}", candidate.display()); + } + if !resolved.is_file() { + bail!("{label} is not a file: {}", candidate.display()); + } + Ok(resolved) +} + +fn resolve_optional_artifact( + base_path: &Path, + value: Option<&str>, + label: &str, +) -> Result<Option<PathBuf>> { + value + .map(|value| resolve_artifact(base_path, value, label)) + .transpose() +} + +fn load_measurement_document<T>( + base_path: &Path, + checksum_file: &Option<Vec<u8>>, + filename: &str, + constructor: impl FnOnce(Vec<u8>, Vec<u8>) -> T, +) -> Result<Option<T>> { + let path = base_path.join(filename); + if !path.exists() { + return Ok(None); + } + let Some(checksum_file) = checksum_file else { + return Ok(None); + }; + let measurement = + fs::read(&path).with_context(|| format!("failed to read {}", path.display()))?; + Ok(Some(constructor(checksum_file.clone(), measurement))) +} + fn guess_version(base_path: &Path) -> Option<String> { // name pattern: dstack-dev-0.2.3 or dstack-0.2.3 let basename = base_path.file_name()?.to_str()?.to_string(); @@ -210,3 +304,92 @@ fn guess_version(base_path: &Path) -> Option<String> { }; Some(version.to_string()) } + +#[cfg(test)] +mod tests { + use super::{Image, ImageInfo, Version}; + use std::{fs, sync::Arc, thread}; + use tempfile::TempDir; + + fn fixture(name: &str, kernel: &str) -> (TempDir, std::path::PathBuf) { + let root = TempDir::new().unwrap(); + let image = root.path().join(name); + fs::create_dir(&image).unwrap(); + let metadata = format!( + r#"{{"cmdline":null,"kernel":"{kernel}","initrd":"initrd","hda":null,"rootfs":null,"bios":null}}"# + ); + fs::write(image.join("metadata.json"), metadata).unwrap(); + fs::write(image.join("initrd"), b"initrd").unwrap(); + (root, image) + } + + #[test] + fn loads_minimal_metadata_and_guesses_legacy_version() { + let (_root, image) = fixture("dstack-dev-1.2.3", "vmlinuz"); + fs::write(image.join("vmlinuz"), b"kernel").unwrap(); + let loaded = Image::load(&image).unwrap(); + assert_eq!(loaded.info.version, "1.2.3"); + assert_eq!(loaded.info.version(), Some(Version::new(1, 2, 3))); + assert!(!loaded.info.shared_ro); + } + + #[test] + fn parses_version_boundaries_and_rejects_missing_artifact() { + for (version, expected) in [ + ("0.0.0", Some(Version::new(0, 0, 0))), + ("65535.1.2", Some(Version::new(65535, 1, 2))), + // A missing patch component defaults to 0. + ("1.2", Some(Version::new(1, 2, 0))), + // A non-numeric patch is not silently coerced to 0. + ("1.2.invalid", None), + // u16 no longer bounds the components. + ("65536.1.2", Some(Version::new(65536, 1, 2))), + // Prerelease and build metadata compare equal to the release. + ("0.6.1-rc1", Some(Version::new(0, 6, 1))), + ("0.6.1.rc1", Some(Version::new(0, 6, 1))), + ("0.6.1+build.5", Some(Version::new(0, 6, 1))), + ("0.6.1-rc.1+build.5", Some(Version::new(0, 6, 1))), + // Still garbage in, None out. + ("", None), + ("nvidia-0.6.0", None), + ("v0.6.1", None), + ] { + let json = format!( + r#"{{"cmdline":null,"kernel":"k","initrd":"i","hda":null,"rootfs":null,"bios":null,"version":"{version}"}}"# + ); + let info: ImageInfo = serde_json::from_str(&json).unwrap(); + assert_eq!(info.version(), expected, "{version}"); + } + let (_root, image) = fixture("missing", "missing"); + assert!(Image::load(image).is_err()); + } + + #[test] + fn rejects_parent_traversal_artifact() { + let (root, image) = fixture("escaped", "../outside"); + fs::write(root.path().join("outside"), b"outside").unwrap(); + assert!(Image::load(image).is_err()); + } + + #[cfg(unix)] + #[test] + fn rejects_symlink_escape_and_concurrent_reads_converge() { + use std::os::unix::fs::symlink; + let (root, image) = fixture("candidate", "kernel-link"); + fs::write(root.path().join("outside"), b"outside").unwrap(); + symlink(root.path().join("outside"), image.join("kernel-link")).unwrap(); + assert!(Image::load(&image).is_err()); + fs::remove_file(image.join("kernel-link")).unwrap(); + fs::write(image.join("kernel-link"), b"kernel").unwrap(); + let image = Arc::new(image); + let workers: Vec<_> = (0..4) + .map(|_| { + let image = Arc::clone(&image); + thread::spawn(move || Image::load(image.as_path()).unwrap().info.version) + }) + .collect(); + for worker in workers { + assert_eq!(worker.join().unwrap(), ""); + } + } +} diff --git a/dstack/vmm/src/app/mr_config.rs b/dstack/vmm/src/app/mr_config.rs index f1b908e4f..e9aa83539 100644 --- a/dstack/vmm/src/app/mr_config.rs +++ b/dstack/vmm/src/app/mr_config.rs @@ -7,11 +7,53 @@ use anyhow::{bail, Context, Result}; use base64::prelude::*; use dstack_types::mr_config::{MrConfig, MrConfigV3}; -use dstack_types::AppCompose; +use dstack_types::{gpu_policy_hash, AppCompose}; use fs_err as fs; use sha2::{Digest, Sha256}; -use super::VmWorkDir; +use super::{GpuConfig, Manifest, VmWorkDir}; +use crate::config::{CvmConfig, CvmPlatform}; + +fn bind_init_script_hashes(mut mr_config: MrConfigV3, app_compose: &AppCompose) -> MrConfigV3 { + if app_compose.manifest_version_u32().unwrap_or_default() >= 3 { + let hashes = app_compose + .init_script + .iter() + .map(|script| Sha256::digest(script.as_bytes()).to_vec()) + .collect(); + mr_config = mr_config.with_init_script_hashes(hashes); + } + mr_config +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum MrConfigVersion { + V1, + V3, +} + +pub(super) fn mr_config_version( + manifest: &Manifest, + platform: CvmPlatform, + use_mrconfigid: bool, + has_key_provider_id: bool, +) -> Result<Option<MrConfigVersion>> { + if manifest.simulated_tee == Some(dstack_types::TeeVariant::DstackAmdSevSnp) { + return Ok(Some(MrConfigVersion::V3)); + } + if manifest.no_tee { + return Ok(None); + } + match platform { + CvmPlatform::AmdSevSnp => Ok(Some(MrConfigVersion::V3)), + CvmPlatform::Tdx if has_key_provider_id && !use_mrconfigid => { + bail!("key provider ID requires MrConfigV3, but use_mrconfigid is disabled") + } + CvmPlatform::Tdx if !use_mrconfigid => Ok(None), + CvmPlatform::Tdx if has_key_provider_id => Ok(Some(MrConfigVersion::V3)), + CvmPlatform::Tdx => Ok(Some(MrConfigVersion::V1)), + } +} pub(super) fn tdx_mr_config_id(workdir: &VmWorkDir, app_compose: &AppCompose) -> Result<String> { if let Some(document) = workdir @@ -60,10 +102,44 @@ pub(super) fn snp_host_data(workdir: &VmWorkDir) -> Result<String> { } impl VmWorkDir { - pub fn prepare_mr_config_v3(&self, app_compose: &AppCompose) -> Result<String> { + pub(crate) fn prepare_mr_config( + &self, + manifest: &Manifest, + config: &CvmConfig, + app_compose: &AppCompose, + ) -> Result<Option<String>> { + let version = mr_config_version( + manifest, + config.resolved_platform(), + config.use_mrconfigid, + !app_compose.key_provider_id.is_empty(), + )?; + match version { + Some(MrConfigVersion::V3) => self + .prepare_mr_config_v3( + app_compose, + manifest.gpus.as_ref().is_some_and(GpuConfig::has_gpus), + ) + .map(Some), + Some(MrConfigVersion::V1) | None => Ok(None), + } + } + + fn prepare_mr_config_v3(&self, app_compose: &AppCompose, has_gpus: bool) -> Result<String> { let compose_hash = self .app_compose_hash() .context("failed to get compose hash")?; + let gpu_policy_hash = if has_gpus { + let compose_json = fs::read(self.app_compose_path()) + .context("failed to read app compose for GPU policy hash")?; + Some( + gpu_policy_hash(&compose_json) + .context("failed to hash raw GPU policy")? + .to_vec(), + ) + } else { + None + }; let mut instance_info = self .instance_info_or_default() .context("failed to get instance info")?; @@ -103,13 +179,61 @@ impl VmWorkDir { ) .context("failed to write instance info")?; - Ok(MrConfigV3::new( + // Manifest v3 is the fail-closed capability signal: older guests + // reject its string version instead of seeing an unknown MrConfigV3 + // field generated by a newer VMM. + let mr_config = MrConfigV3::new( app_id, compose_hash.to_vec(), + gpu_policy_hash, app_compose.key_provider(), app_compose.key_provider_id.clone(), instance_id, + ); + Ok(bind_init_script_hashes(mr_config, app_compose).to_canonical_json()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use dstack_types::KeyProviderKind; + + fn app_compose(manifest_version: serde_json::Value) -> AppCompose { + serde_json::from_value(serde_json::json!({ + "manifest_version": manifest_version, + "name": "test", + "runner": "docker-compose" + })) + .unwrap() + } + + fn base_mr_config() -> MrConfigV3 { + MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + None, + KeyProviderKind::None, + Vec::new(), + vec![0x33; 20], ) - .to_canonical_json()) + } + + #[test] + fn manifest_v2_omits_init_script_hashes() { + let document = + bind_init_script_hashes(base_mr_config(), &app_compose(serde_json::json!(2))) + .to_canonical_json(); + let document: serde_json::Value = serde_json::from_str(&document).unwrap(); + assert!(document.get("init_script_hashes").is_none()); + } + + #[test] + fn manifest_v3_includes_empty_init_script_hashes() { + let document = + bind_init_script_hashes(base_mr_config(), &app_compose(serde_json::json!("3"))) + .to_canonical_json(); + let document: serde_json::Value = serde_json::from_str(&document).unwrap(); + assert_eq!(document["init_script_hashes"], serde_json::json!([])); } } diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs index 2ed379423..b946e3ca1 100644 --- a/dstack/vmm/src/app/network.rs +++ b/dstack/vmm/src/app/network.rs @@ -9,73 +9,250 @@ use std::path::Path; use anyhow::{bail, Result}; use sha2::{Digest, Sha256}; -use super::Manifest; -use crate::config::{CvmConfig, Networking, NetworkingMode}; +use super::{Manifest, PortMapping}; +use crate::config::{ + CvmConfig, NetworkFilterMode, Networking, NetworkingMode, NicNetworking, MAX_NET_QUEUES, +}; -pub(crate) fn resolve_networking(networking: &Networking, cfg: &CvmConfig) -> Networking { +/// Node configuration merged with what one NIC pins. +/// +/// The node half is taken wholesale and the NIC half overrides it where it is +/// set. Nothing the node owns -- the macvtap forwarding mode, the MAC prefix, +/// the user-mode network parameters, a custom netdev string -- can be +/// overridden here any more, because a [`NicNetworking`] cannot carry one. +pub(crate) fn resolve_networking( + networking: &NicNetworking, + cfg: &CvmConfig, + vcpu: u32, +) -> Networking { let mut resolved = cfg.networking.clone(); - resolved.mode = networking.mode; - resolved.restrict = cfg.networking.restrict || networking.restrict; + // A deployment that only tuned the data plane never named a backend, so + // the node keeps deciding which one this NIC uses -- including after the + // operator changes it. + resolved.nic.inherit_mode = networking.inherit_mode; + resolved.nic.mode = if networking.inherit_mode { + cfg.networking.nic.mode + } else { + networking.mode + }; + // Runtime state, never inherited from configuration or from a previous + // launch. Interface preparation sets both for the NICs it builds, and a + // node configuration that names either is rejected at startup. + resolved.device.clear(); if !networking.bridge.is_empty() { - resolved.bridge = networking.bridge.clone(); - } - if !networking.mac_prefix.is_empty() { - resolved.mac_prefix = networking.mac_prefix.clone(); + resolved.nic.bridge = networking.bridge.clone(); } - if !networking.net.is_empty() { - resolved.net = networking.net.clone(); + if !networking.parent.is_empty() { + resolved.nic.parent = networking.parent.clone(); } - if !networking.dhcp_start.is_empty() { - resolved.dhcp_start = networking.dhcp_start.clone(); - } - if !networking.netdev.is_empty() { - resolved.netdev = networking.netdev.clone(); + if networking.vhost.is_some() { + resolved.nic.vhost = networking.vhost; } + // Make the vCPU-scaled default concrete here, so every later stage -- + // netd preparation, the QEMU arguments, and removal after a VMM restart -- + // reads one number instead of recomputing it from a vCPU count it may no + // longer have. + resolved.nic.queues = Some(match networking.queues { + // An explicit count is honoured whatever the data plane: multiqueue + // without vhost is a valid, if unusual, thing to ask for. + Some(queues) => queues, + // Without vhost the QEMU main loop drains every queue on one thread, + // so scaling up buys almost nothing while still costing a netd + // interface, extra vectors, and a changed device. Anyone turning vhost + // off is asking for the old data plane; give them the old shape too. + None if resolved.vhost_enabled() => { + Networking::default_queue_pairs(vcpu, cfg.max_net_queues) + } + None => 1, + }); resolved } pub(crate) fn resolved_networks(manifest: &Manifest, cfg: &CvmConfig) -> Vec<Networking> { - if manifest.networks.is_empty() { - vec![cfg.networking.clone()] + let node_default = [cfg.networking.nic.clone()]; + let requested = if manifest.networks.is_empty() { + &node_default[..] } else { - manifest - .networks - .iter() - .map(|networking| resolve_networking(networking, cfg)) - .collect() + &manifest.networks[..] + }; + requested + .iter() + .map(|networking| resolve_networking(networking, cfg, manifest.vcpu)) + .collect() +} + +/// Whether netd must pre-create the host interface for this NIC. +/// +/// Every macvtap and every bridge NIC. The alternative was a set of conditions +/// -- libvirt filtering, multiqueue -- under which netd was consulted and +/// outside of which the VMM built the interface some other way. Each of those +/// paths had to answer the same questions again and answer them differently: +/// which netdev QEMU gets, whether vhost is really on, and, once port mappings +/// grew a NIC, where a bridge NIC's host ports go. A host interface has one +/// owner now, and `port_map` on a bridge reaches netd on every node rather than +/// only on the ones that happened to filter or to have scaled their queues. +/// +/// The cost is stated plainly: bridge and macvtap need a netd on the host. User +/// mode and a caller-supplied netdev still need nothing. +pub(crate) fn needs_netd_interface(networking: &Networking) -> bool { + matches!( + networking.nic.mode, + NetworkingMode::Macvtap | NetworkingMode::Bridge + ) +} + +/// Whether this NIC's host interface carries a libvirt nwfilter binding. +/// Macvtap never does, and a bridge NIC only does when the node filters. +pub(crate) fn filters_bridge_traffic(networking: &Networking, cfg: &CvmConfig) -> bool { + networking.nic.mode == NetworkingMode::Bridge + && cfg.network_filter.mode == NetworkFilterMode::Libvirt +} + +/// Makes the data plane concrete on a launch-time NIC list. +/// +/// `vhost` on a freshly resolved entry is still a *request*: `None` means +/// inherit from the node, which can change under a VM that is already running. +/// Settling it once, here, is what lets the QEMU arguments and the reported +/// status read the same value for the life of a boot. +pub(crate) fn settle_vhost(networks: &mut [Networking]) { + for networking in networks.iter_mut() { + networking.nic.vhost = Some(networking.vhost_enabled()); } } pub(crate) fn validate_resolved_network(networking: &Networking) -> Result<()> { - if networking.mode != NetworkingMode::Bridge { + // The vCPU-scaled default is bounded by construction; only an explicit + // request can exceed the hard cap. + if networking + .nic + .queues + .is_some_and(|queues| queues > MAX_NET_QUEUES) + { + bail!("networking queues must not exceed {MAX_NET_QUEUES}"); + } + if networking.nic.mode != NetworkingMode::Bridge { return Ok(()); } - if networking.bridge.is_empty() { + if networking.nic.bridge.is_empty() { bail!("bridge networking requested but no bridge is configured"); } if !Path::new("/sys/class/net") - .join(&networking.bridge) + .join(&networking.nic.bridge) .exists() { - bail!("bridge interface '{}' does not exist", networking.bridge); + bail!( + "bridge interface '{}' does not exist", + networking.nic.bridge + ); } Ok(()) } pub(crate) fn validate_resolved_networks(networks: &[Networking]) -> Result<()> { - let forwarding_bridges = networks - .iter() - .filter(|networking| networking.is_bridge() && networking.forward_service_enabled) - .count(); - if forwarding_bridges > 1 { - bail!("built-in port forwarding supports only one bridge"); - } for networking in networks { validate_resolved_network(networking)?; } Ok(()) } +/// Warns when a vhost NIC is about to launch on a host that has no +/// `/dev/vhost-net` at all. +/// +/// QEMU exits when `vhost=on` cannot open the device, and it does so from +/// inside the per-VM launcher where the reason is easy to miss. This puts the +/// remediation in the VMM log instead. +/// +/// Both checks are warnings, never refusals. QEMU is not necessarily this +/// process — an externally started supervisor can run it under another account +/// — so neither answers the question that decides the launch. They are the two +/// cheap statements that catch the two ways this actually goes wrong. +/// +/// The permission check matters because the node's mode is not uniform. It is +/// `root:kvm 0660` on Debian-family hosts, where a VMM in the `kvm` group is +/// fine, and `root:root 0600` on several others — where every bridge VM stops +/// restarting after an upgrade turns vhost on node-wide, with the only +/// explanation buried in a per-VM launcher's QEMU output. Existence alone says +/// nothing about that case, which is the likelier of the two. +pub(crate) fn warn_if_vhost_net_missing(networks: &[Networking]) { + const VHOST_NET: &str = "/dev/vhost-net"; + if !networks.iter().any(Networking::vhost_enabled) { + return; + } + // The node is a kmod static device node, so it is present even before + // vhost_net is loaded; QEMU's open autoloads the module. + if !Path::new(VHOST_NET).exists() { + tracing::warn!( + "{VHOST_NET} is missing; vhost networking will fail to start. load the vhost_net \ + module, or set vhost = false in [cvm.networking]" + ); + return; + } + if let Err(error) = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(VHOST_NET) + { + if error.kind() == std::io::ErrorKind::PermissionDenied { + tracing::warn!( + "{VHOST_NET} is not accessible to this process; if QEMU runs under the same \ + account, vhost networking will fail to start. add that account to the device's \ + group, or set vhost = false in [cvm.networking]" + ); + } + } +} + +/// Which NIC an unpinned port mapping's traffic enters through. +/// +/// The first user-mode NIC, which is where QEMU's `hostfwd=` entries have +/// always gone. There is no second choice: nothing else on this host publishes +/// a port, so a VM without one has nowhere to put a mapping and the launch +/// says so. +pub(crate) fn default_ingress_nic(networks: &[Networking]) -> Option<usize> { + networks + .iter() + .position(|network| network.nic.mode == NetworkingMode::User) +} + +/// Whether a NIC of this mode has a mechanism to publish a host port at all. +/// +/// QEMU's `hostfwd=`, and nothing else. A bridge TAP is built by netd, and the +/// netd in this repository does not forward host ports; macvtap bypasses the +/// host bridge; a custom netdev is a string the VMM does not interpret. +pub(crate) fn mode_carries_ingress(mode: NetworkingMode) -> bool { + matches!(mode, NetworkingMode::User) +} + +/// Which NIC a port mapping's traffic enters through. +/// +/// One mapping resolves to at most one NIC, and only a user-mode NIC has a +/// mechanism to carry it, so a pin to any other kind resolves to nothing +/// rather than to a NIC with no path into the guest. +/// +/// `None` is a mapping with nowhere to go: a VM with no user-mode NIC, or one +/// whose NICs changed under a mapping that named one. The launch warns about +/// each of those rather than dropping it in silence. +pub(crate) fn ingress_nic(mapping: &PortMapping, networks: &[Networking]) -> Option<usize> { + mapping + .nic_index + .or_else(|| default_ingress_nic(networks)) + .filter(|index| { + networks + .get(*index) + .is_some_and(|network| mode_carries_ingress(network.nic.mode)) + }) +} + +/// Names the mappings that resolve to no NIC, for a launch to warn about. +pub(crate) fn stranded_ingress<'a>( + port_map: &'a [PortMapping], + networks: &'a [Networking], +) -> impl Iterator<Item = &'a PortMapping> { + port_map + .iter() + .filter(|mapping| ingress_nic(mapping, networks).is_none()) +} + /// Derives a deterministic, locally administered unicast MAC address. /// /// Index zero preserves the legacy single-NIC derivation. Later interfaces @@ -102,7 +279,241 @@ pub(crate) fn mac_address_for_vm_index(vm_id: &str, prefix: &[u8], index: usize) #[cfg(test)] mod tests { - use super::mac_address_for_vm_index; + use super::{ + default_ingress_nic, ingress_nic, mac_address_for_vm_index, needs_netd_interface, + resolved_networks, settle_vhost, stranded_ingress, validate_resolved_networks, + }; + use crate::app::PortMapping; + use crate::config::Protocol; + use crate::config::{Networking, NetworkingMode, NicNetworking}; + + fn macvtap_network() -> NicNetworking { + NicNetworking { + mode: NetworkingMode::Macvtap, + parent: "eth0".into(), + // Most tests here exercise the vhost data plane, which the + // shipped default leaves off; opt in the way a real node would. + vhost: Some(true), + ..NicNetworking::default() + } + } + + /// The same NIC as a resolved value, for the checks that run against what + /// a launch would see rather than against what a VM pins. + fn macvtap_resolved() -> Networking { + Networking { + nic: macvtap_network(), + ..Networking::default() + } + } + + fn node_config(mode: NetworkingMode) -> crate::config::CvmConfig { + use rocket::figment::providers::Format as _; + let config: crate::config::Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + let mut cvm = config.cvm; + cvm.networking.nic.mode = mode; + cvm.networking.nic.bridge = "br0".into(); + cvm.networking.nic.parent = "eth0".into(); + // The shipped default leaves vhost off; the tests here are about the + // vhost data plane, so this node opts in the way a real one would. + cvm.networking.nic.vhost = Some(true); + cvm + } + + /// A node whose configuration never names `vhost` — the upgrade case, + /// where the toml predates the key entirely. + fn unconfigured_vhost_node(mode: NetworkingMode) -> crate::config::CvmConfig { + let mut cvm = node_config(mode); + cvm.networking.nic.vhost = None; + cvm + } + + fn manifest_with(vcpu: u32, networks: Vec<NicNetworking>) -> crate::app::Manifest { + let mut manifest: crate::app::Manifest = serde_json::from_value(serde_json::json!({ + "id": "vm-1", "name": "n", "app_id": "a", "vcpu": vcpu, "memory": 2048, + "disk_size": 10, "image": "i", "port_map": [], "created_at_ms": 0, + })) + .unwrap(); + manifest.networks = networks; + manifest + } + + #[test] + fn queue_pairs_default_to_the_vcpu_count_up_to_the_cap() { + let cvm = node_config(NetworkingMode::Bridge); + for (vcpu, want) in [(1, 1), (2, 2), (8, 8), (16, 16), (32, 16), (128, 16)] { + let resolved = resolved_networks(&manifest_with(vcpu, vec![]), &cvm); + assert_eq!( + resolved[0].queue_pairs(), + want, + "vcpu {vcpu} should give {want} queue pairs" + ); + } + } + + /// An upgraded node must keep building the device its VMs have always + /// had: userspace virtio, one queue pair. Both the toml that predates the + /// `vhost` key and the shipped default say so. + #[test] + fn a_node_that_never_asked_for_vhost_keeps_the_old_device_shape() { + let shipped = { + use rocket::figment::providers::Format as _; + let config: crate::config::Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + config.cvm.networking.nic.vhost + }; + for vhost in [None, shipped] { + let mut cvm = unconfigured_vhost_node(NetworkingMode::Bridge); + cvm.networking.nic.vhost = vhost; + let resolved = resolved_networks(&manifest_with(16, vec![]), &cvm); + assert!(!resolved[0].vhost_enabled()); + assert_eq!(resolved[0].queue_pairs(), 1); + } + } + + #[test] + fn turning_vhost_off_also_turns_off_the_multiqueue_default() { + let mut cvm = node_config(NetworkingMode::Bridge); + cvm.networking.nic.vhost = Some(false); + let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert!(!resolved[0].vhost_enabled()); + assert_eq!(resolved[0].queue_pairs(), 1); + + // Per-VM opt-out does the same thing. + let cvm = node_config(NetworkingMode::Bridge); + let mut asked = cvm.networking.nic.clone(); + asked.vhost = Some(false); + let resolved = resolved_networks(&manifest_with(8, vec![asked]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 1); + + // But an explicit queue count is still honoured without vhost. + let mut asked = cvm.networking.nic.clone(); + asked.vhost = Some(false); + asked.queues = Some(4); + let resolved = resolved_networks(&manifest_with(8, vec![asked]), &cvm); + assert!(!resolved[0].vhost_enabled()); + assert_eq!(resolved[0].queue_pairs(), 4); + } + + #[test] + fn lowering_the_request_ceiling_also_lowers_the_default() { + let mut cvm = node_config(NetworkingMode::Bridge); + cvm.max_net_queues = 2; + let resolved = resolved_networks(&manifest_with(16, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 2); + + // Raising it past the scaling cap widens requests, not the default. + cvm.max_net_queues = 32; + let resolved = resolved_networks(&manifest_with(24, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 16); + } + + /// One owner for a host interface, with no condition attached. What used + /// to decide this -- libvirt filtering, a scaled queue count -- decided it + /// per node, so the same VM definition got a netd-built TAP on one host and + /// a `qemu-bridge-helper` TAP on the next. + #[test] + fn every_bridge_and_macvtap_nic_is_netds_to_build() { + let mut cvm = node_config(NetworkingMode::Bridge); + let mut single = cvm.networking.nic.clone(); + single.queues = Some(1); + let resolved = resolved_networks(&manifest_with(8, vec![single.clone()]), &cvm); + assert!(needs_netd_interface(&resolved[0])); + + // Unfiltered, single queue, no vhost -- the shape that used to need no + // netd at all -- is netd's too. + cvm.networking.nic.vhost = Some(false); + single.vhost = Some(false); + let resolved = resolved_networks(&manifest_with(8, vec![single]), &cvm); + assert!(!resolved[0].vhost_enabled()); + assert_eq!(resolved[0].queue_pairs(), 1); + assert!(needs_netd_interface(&resolved[0])); + + let cvm = node_config(NetworkingMode::Macvtap); + let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert!(needs_netd_interface(&resolved[0])); + + // The two backends the VMM builds itself still need nothing. + for mode in [NetworkingMode::User, NetworkingMode::Custom] { + let cvm = node_config(mode); + let resolved = resolved_networks(&manifest_with(8, vec![]), &cvm); + assert!(!needs_netd_interface(&resolved[0])); + } + } + + #[test] + fn an_explicit_queue_count_survives_resolution() { + let cvm = node_config(NetworkingMode::Bridge); + let mut asked = macvtap_network(); + asked.mode = NetworkingMode::Bridge; + asked.queues = Some(2); + let resolved = resolved_networks(&manifest_with(16, vec![asked]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 2); + } + + #[test] + fn user_mode_stays_single_queue_whatever_the_vcpu_count() { + let cvm = node_config(NetworkingMode::User); + let resolved = resolved_networks(&manifest_with(32, vec![]), &cvm); + assert_eq!(resolved[0].queue_pairs(), 1); + } + + #[test] + fn validation_never_depends_on_this_process_reaching_vhost_net() { + // QEMU may run under different credentials, so a NIC that asks for + // vhost must validate on hosts where the VMM itself cannot open the + // device. Both of these hold whether or not /dev/vhost-net exists here. + let mut networking = macvtap_resolved(); + assert!(networking.vhost_enabled()); + validate_resolved_networks(&[networking.clone()]).unwrap(); + + networking.nic.queues = Some(4); + validate_resolved_networks(&[networking]).unwrap(); + } + + #[test] + fn queue_counts_above_the_hard_bound_are_rejected() { + let mut networking = macvtap_resolved(); + networking.nic.queues = Some(super::MAX_NET_QUEUES + 1); + let error = validate_resolved_networks(&[networking]).unwrap_err(); + assert!(error.to_string().contains("must not exceed")); + } + + /// The data plane a NIC actually gets is decided once, at launch, and + /// written into the runtime entry. Recomputing it later would let a report + /// about a running VM change under an operator's edit to node + /// configuration, describing a data plane QEMU is not using. + #[test] + fn settling_vhost_records_what_the_launch_decided() { + let mut cvm = node_config(NetworkingMode::Bridge); + let manifest = manifest_with(8, vec![]); + let mut networks = resolved_networks(&manifest, &cvm); + assert_eq!(networks[0].nic.vhost, Some(true)); + settle_vhost(&mut networks); + assert_eq!(networks[0].nic.vhost, Some(true)); + + // The node turns vhost off under a VM that is already running. The + // entry the launch settled keeps its answer; the next boot gets the + // new one. + cvm.networking.nic.vhost = Some(false); + assert_eq!(networks[0].nic.vhost, Some(true)); + let mut next_boot = resolved_networks(&manifest, &cvm); + settle_vhost(&mut next_boot); + assert_eq!(next_boot[0].nic.vhost, Some(false)); + + // An inherited `None` becomes a decision rather than staying a request. + let mut unset = resolved_networks(&manifest, &cvm); + unset[0].nic.vhost = None; + settle_vhost(&mut unset); + assert_eq!(unset[0].nic.vhost, Some(false)); + } #[test] fn primary_mac_keeps_legacy_derivation_and_later_nics_are_distinct() { @@ -115,4 +526,80 @@ mod tests { "c6:74:2c:65:14:b9" ); } + + fn nic(mode: NetworkingMode) -> Networking { + Networking { + nic: NicNetworking { + mode, + ..NicNetworking::default() + }, + ..Networking::default() + } + } + + fn mapping(host_port: u16, nic_index: Option<usize>) -> PortMapping { + PortMapping { + address: "0.0.0.0".parse().unwrap(), + protocol: Protocol::Tcp, + from: host_port, + to: host_port, + nic_index, + } + } + + #[test] + fn an_unpinned_mapping_still_lands_where_hostfwd_always_put_it() { + // Existing VMs must not move. QEMU's `hostfwd=` has always gone to the + // first user-mode NIC, so that stays the answer wherever there is one. + let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::User)]; + assert_eq!(default_ingress_nic(&networks), Some(1)); + assert_eq!(ingress_nic(&mapping(443, None), &networks), Some(1)); + + // With no user-mode NIC there is nowhere at all. netd builds a bridge + // TAP but does not forward host ports, and macvtap and custom have no + // path either. + let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::Bridge)]; + assert_eq!(default_ingress_nic(&networks), None); + + let networks = [nic(NetworkingMode::Macvtap), nic(NetworkingMode::Custom)]; + assert_eq!(default_ingress_nic(&networks), None); + assert_eq!(ingress_nic(&mapping(443, None), &networks), None); + } + + #[test] + fn a_pinned_mapping_goes_where_it_says() { + let networks = [nic(NetworkingMode::Bridge), nic(NetworkingMode::User)]; + assert_eq!(ingress_nic(&mapping(443, Some(1)), &networks), Some(1)); + // Out of range resolves to nothing rather than to something arbitrary. + // Deployment refuses it outright; a manifest that lost a NIC lands here. + assert_eq!(ingress_nic(&mapping(443, Some(7)), &networks), None); + } + + /// A pin has to be checked against the backend, not just the count. + /// Naming a macvtap or custom NIC used to resolve to that index and then + /// fall out of every branch that could act on it: no `hostfwd=`, no netd + /// request, and no warning either. + #[test] + fn a_pin_to_a_backend_with_no_ingress_resolves_to_nothing() { + let networks = [ + nic(NetworkingMode::Macvtap), + nic(NetworkingMode::Custom), + nic(NetworkingMode::Bridge), + nic(NetworkingMode::User), + ]; + assert_eq!(ingress_nic(&mapping(443, Some(0)), &networks), None); + assert_eq!(ingress_nic(&mapping(443, Some(1)), &networks), None); + // A bridge TAP is netd's, and netd does not forward host ports. + assert_eq!(ingress_nic(&mapping(443, Some(2)), &networks), None); + assert_eq!(ingress_nic(&mapping(443, Some(3)), &networks), Some(3)); + + // And an unpinned mapping on a VM with nowhere to put it is named, + // rather than counted as delivered. + let networks = [nic(NetworkingMode::Macvtap)]; + let port_map = [mapping(443, None), mapping(8080, Some(0))]; + let stranded: Vec<_> = stranded_ingress(&port_map, &networks) + .map(|mapping| mapping.from) + .collect(); + assert_eq!(stranded, vec![443, 8080]); + } } diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 6850bb653..f0652dd13 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -3,33 +3,37 @@ // SPDX-License-Identifier: Apache-2.0 //! QEMU launch preparation and command construction. -use crate::{ - app::Manifest, - config::{CvmConfig, Networking, NetworkingMode, ProcessAnnotation, TeePlatform}, -}; -use std::collections::HashMap; -use std::os::unix::fs::PermissionsExt; -use std::{ - fs::Permissions, - io::Write, - path::{Path, PathBuf}, - process::{Command, Stdio}, -}; - use super::{ effective_vcpu_count, host_share::create_shared_disk, hugepage_numa_nodes, image::Image, mr_config::{snp_host_data, tdx_mr_config_id}, - network::{mac_address_for_vm_index, resolved_networks, validate_resolved_networks}, + network::{ + ingress_nic, mac_address_for_vm_index, validate_resolved_networks, + warn_if_vhost_net_missing, + }, pci_numa_node, round_up, GpuConfig, VmWorkDir, }; +use crate::{ + app::Manifest, + config::{CvmConfig, CvmPlatform, Networking, NetworkingMode, ProcessAnnotation}, + netd::{tap_name, InterfaceIdentity}, + vm_launcher::{ChildCommand, LaunchSpec, OpenFile}, +}; use anyhow::{bail, Context, Result}; use bon::Builder; -use dstack_types::{shared_filenames::HOST_SHARED_DISK_LABEL, KeyProviderKind}; +use dstack_types::shared_filenames::HOST_SHARED_DISK_LABEL; +use dstack_types::version::Version; use fs_err as fs; +use nix::unistd::{Gid, Uid}; use serde::Serialize; +use std::collections::HashMap; +use std::{ + io::Write, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; use supervisor_client::supervisor::ProcessConfig; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -164,6 +168,14 @@ fn create_hd( Ok(()) } +fn on_off(enabled: bool) -> &'static str { + if enabled { + "on" + } else { + "off" + } +} + fn virtio_pci_device(device: &str, snp: bool) -> String { if snp { format!("{device},disable-legacy=on,iommu_platform=true") @@ -172,14 +184,49 @@ fn virtio_pci_device(device: &str, snp: bool) -> String { } } +struct PreparedVolume { + source: String, +} + +/// First descriptor the per-VM launcher may hand to QEMU. Zero through two are +/// the standard streams. +const FIRST_INHERITED_FD: i32 = 3; + +/// Descriptors the launcher opens for each macvtap NIC, one per queue pair. +/// +/// Both the launcher's open list and the `-netdev` arguments derive from this +/// one layout, so they cannot disagree about which descriptor belongs to which +/// NIC. +fn macvtap_fd_layout(networks: &[Networking]) -> Vec<Vec<i32>> { + let mut next_fd = FIRST_INHERITED_FD; + networks + .iter() + .map(|network| { + if network.nic.mode != NetworkingMode::Macvtap { + return Vec::new(); + } + (0..network.queue_pairs()) + .map(|_| { + let fd = next_fd; + next_fd += 1; + fd + }) + .collect() + }) + .collect() +} + struct PreparedQemuLaunch { workdir: VmWorkDir, - platform: TeePlatform, + platform: CvmPlatform, networks: Vec<Networking>, + volumes: Vec<PreparedVolume>, + storage_discard: bool, hugepage_numa_nodes: Option<HashMap<String, u32>>, gpu_numa_nodes: HashMap<String, String>, numa_cpus: Option<String>, - tpm_path: Option<&'static str>, + swtpm_socket: Option<PathBuf>, + swtpm_path: Option<PathBuf>, tdx_mr_config_id: Option<String>, snp_host_data: Option<String>, snp_launch_params: Option<AmdSevSnpLaunchParams>, @@ -191,14 +238,24 @@ impl PreparedQemuLaunch { workdir: impl AsRef<Path>, cfg: &CvmConfig, gpus: &GpuConfig, + networks: &[Networking], ) -> Result<Self> { let workdir = VmWorkDir::new(workdir); - prepare_data_disk(vm, &workdir, cfg)?; + prepare_data_disk(vm, &workdir)?; prepare_shared_dir(&workdir)?; let app_compose = workdir.app_compose().context("failed to get app compose")?; let platform = cfg.resolved_platform(); - let networks = resolved_networks(&vm.manifest, cfg); + let networks = networks.to_vec(); validate_resolved_networks(&networks)?; + warn_if_vhost_net_missing(&networks); + let volumes = vm + .manifest + .volumes + .iter() + .map(|volume| PreparedVolume { + source: volume.source.clone(), + }) + .collect(); let hugepage_numa_nodes = if vm.manifest.hugepages { Some(hugepage_numa_nodes(gpus)?) @@ -219,25 +276,33 @@ impl PreparedQemuLaunch { } else { None }; - let tpm_path = if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) { - Some(detect_tpm_device()?) + let (swtpm_socket, swtpm_path) = if vm.manifest.swtpm { + let swtpm_path = which::which("swtpm") + .context("tpm key provider requested but swtpm is not installed")?; + let state_dir = workdir.swtpm_state_dir(); + fs::create_dir_all(&state_dir).context("failed to create swtpm state directory")?; + let socket = workdir.swtpm_socket(); + if socket.exists() { + fs::remove_file(&socket).context("failed to remove stale swtpm socket")?; + } + (Some(socket), Some(swtpm_path)) } else { - None + (None, None) }; prepare_shared_disk(&workdir, cfg)?; let tee_enabled = !vm.manifest.no_tee; let tdx_mr_config_id = if tee_enabled - && platform == TeePlatform::Tdx + && platform == CvmPlatform::Tdx && cfg.use_mrconfigid - && vm.image.info.version_tuple().unwrap_or_default() >= (0, 5, 2) + && vm.image.info.version().unwrap_or_default() >= Version::new(0, 5, 2) { Some(tdx_mr_config_id(&workdir, &app_compose)?) } else { None }; let (snp_host_data, snp_launch_params) = - if tee_enabled && platform == TeePlatform::AmdSevSnp { + if tee_enabled && platform == CvmPlatform::AmdSevSnp { ( Some(snp_host_data(&workdir)?), Some( @@ -254,10 +319,13 @@ impl PreparedQemuLaunch { workdir, platform, networks, + volumes, + storage_discard: app_compose.storage_discard, hugepage_numa_nodes, gpu_numa_nodes, numa_cpus, - tpm_path, + swtpm_socket, + swtpm_path, tdx_mr_config_id, snp_host_data, snp_launch_params, @@ -265,7 +333,7 @@ impl PreparedQemuLaunch { } } -fn prepare_data_disk(vm: &VmConfig, workdir: &VmWorkDir, cfg: &CvmConfig) -> Result<()> { +fn prepare_data_disk(vm: &VmConfig, workdir: &VmWorkDir) -> Result<()> { let hda_path = workdir.hda_path(); if !hda_path.exists() { create_hd( @@ -274,9 +342,6 @@ fn prepare_data_disk(vm: &VmConfig, workdir: &VmWorkDir, cfg: &CvmConfig) -> Res &format!("{}G", vm.manifest.disk_size), )?; } - if !cfg.user.is_empty() { - fs::set_permissions(&hda_path, Permissions::from_mode(0o660))?; - } Ok(()) } @@ -301,16 +366,6 @@ fn prepare_shared_disk(workdir: &VmWorkDir, cfg: &CvmConfig) -> Result<()> { create_shared_disk(&shared_disk_path, shared_dir).context("failed to create shared disk") } -fn detect_tpm_device() -> Result<&'static str> { - if Path::new("/dev/tpmrm0").exists() { - Ok("/dev/tpmrm0") - } else if Path::new("/dev/tpm0").exists() { - Ok("/dev/tpm0") - } else { - bail!("tpm key provider requested but no TPM device found on host") - } -} - struct QemuCommandBuilder<'a> { vm: &'a VmConfig, cfg: &'a CvmConfig, @@ -324,8 +379,9 @@ impl VmConfig { workdir: impl AsRef<Path>, cfg: &CvmConfig, gpus: &GpuConfig, + networks: &[Networking], ) -> Result<Vec<ProcessConfig>> { - let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus)?; + let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus, networks)?; let process = QemuCommandBuilder { vm: self, cfg, @@ -333,7 +389,100 @@ impl VmConfig { prepared: &prepared, } .build()?; - Ok(vec![process]) + let has_macvtap = prepared + .networks + .iter() + .any(|network| network.nic.mode == NetworkingMode::Macvtap); + let Some(socket) = prepared.swtpm_socket.as_deref() else { + if has_macvtap { + return self.wrap_launcher(&prepared, process, None, None); + } + return Ok(vec![process]); + }; + let swtpm_path = prepared + .swtpm_path + .as_ref() + .context("missing swtpm executable for configured socket")?; + let (socket_uid, socket_gid) = (Uid::effective().as_raw(), Gid::effective().as_raw()); + + let swtpm_args = vec![ + "socket".into(), + "--tpm2".into(), + "--tpmstate".into(), + format!("dir={}", prepared.workdir.swtpm_state_dir().display()), + "--ctrl".into(), + format!( + "type=unixio,path={},mode=0600,uid={socket_uid},gid={socket_gid}", + socket.display() + ), + "--flags".into(), + "not-need-init,startup-clear".into(), + ]; + self.wrap_launcher( + &prepared, + process, + Some(ChildCommand { + command: swtpm_path.to_string_lossy().into_owned(), + args: swtpm_args, + }), + Some(socket.to_path_buf()), + ) + } + + fn wrap_launcher( + &self, + prepared: &PreparedQemuLaunch, + process: ProcessConfig, + swtpm: Option<ChildCommand>, + swtpm_socket: Option<PathBuf>, + ) -> Result<Vec<ProcessConfig>> { + // Each queue pair is a separate open of the same macvtap character + // device; the kernel attaches one tap queue per open. + let open_files = prepared + .networks + .iter() + .zip(macvtap_fd_layout(&prepared.networks)) + .flat_map(|(network, fds)| { + fds.into_iter().map(|fd| OpenFile { + fd, + path: network.device.clone().into(), + }) + }) + .collect(); + let spec = LaunchSpec { + qemu: ChildCommand { + command: process.command, + args: process.args, + }, + swtpm, + swtpm_socket, + open_files, + startup_timeout_ms: 5_000, + shutdown_timeout_ms: 10_000, + }; + let spec_path = prepared.workdir.launch_spec_path(); + safe_write::safe_write(&spec_path, serde_json::to_vec_pretty(&spec)?) + .context("failed to write VM launch specification")?; + let executable = + std::env::current_exe().context("failed to locate dstack-vmm executable")?; + let launcher = ProcessConfig { + id: self.manifest.id.clone(), + name: self.manifest.name.clone(), + command: executable.to_string_lossy().into_owned(), + args: vec![ + "vm-launcher".into(), + "--spec".into(), + spec_path.to_string_lossy().into_owned(), + ], + env: process.env, + cwd: process.cwd, + stdout: process.stdout, + stderr: process.stderr, + pidfile: process.pidfile, + cid: process.cid, + note: process.note, + }; + Ok(vec![launcher]) } } @@ -342,6 +491,7 @@ impl QemuCommandBuilder<'_> { let mut command = self.base_command(); self.configure_rootfs(&mut command)?; self.configure_data_disk(&mut command); + self.configure_volumes(&mut command); self.configure_networking(&mut command)?; self.vm.configure_smbios(&mut command, self.cfg); self.configure_tpm_and_vsock(&mut command); @@ -363,7 +513,7 @@ impl QemuCommandBuilder<'_> { } fn is_amd_sev_snp(&self) -> bool { - self.prepared.platform == TeePlatform::AmdSevSnp && !self.vm.manifest.no_tee + self.prepared.platform == CvmPlatform::AmdSevSnp && !self.vm.manifest.no_tee } fn base_command(&self) -> Command { @@ -377,8 +527,13 @@ impl QemuCommandBuilder<'_> { }); command.arg("-nographic"); command.arg("-nodefaults"); + // logappend=on stops QEMU from truncating the log when it opens the + // chardev, which is what makes in-place rotation safe: the fd is + // O_APPEND, so writes resume at the end of file after we truncate. + // Without it QEMU keeps writing at its old offset and punches a sparse + // hole instead, leaving the file as large as it was. command.arg("-chardev").arg(format!( - "pty,id=com0,path={},logfile={}", + "pty,id=com0,path={},logfile={},logappend=on", workdir.serial_pty().display(), workdir.serial_file().display() )); @@ -416,21 +571,17 @@ impl QemuCommandBuilder<'_> { let Some(rootfs) = &self.vm.image.rootfs else { return Ok(()); }; - let image_version = self.vm.image.info.version_tuple().unwrap_or_default(); let extension = rootfs .extension() .unwrap_or_default() .to_str() .unwrap_or_default(); match extension { - "iso" => { - if image_version >= (0, 5, 0) { - bail!( - "Unsupported rootfs type: {extension}. Image versions >= 0.5.0 must use verity rootfs" - ); - } - command.arg("-cdrom").arg(rootfs); - } + // Images before 0.5.0 shipped an `.iso` rootfs booted via `-cdrom`, + // with no dm-verity behind it. `make_sys_config` has rejected those + // images since it started requiring >= 0.5.0, so that branch was + // already unreachable; dropping it keeps the rejection explicit + // instead of leaving a non-verity boot path one edit away. "verity" => { command.arg("-drive").arg(format!( "file={},if=none,id=hd0,format=raw,readonly=on", @@ -450,8 +601,13 @@ impl QemuCommandBuilder<'_> { command .arg("-drive") .arg(format!( - "file={},if=none,id=hd1", - self.prepared.workdir.hda_path().display() + "file={},if=none,id=hd1,discard={}", + self.prepared.workdir.hda_path().display(), + if self.prepared.storage_discard { + "unmap" + } else { + "ignore" + } )) .arg("-device") .arg(virtio_pci_device( @@ -460,12 +616,28 @@ impl QemuCommandBuilder<'_> { )); } + fn configure_volumes(&self, command: &mut Command) { + // Sources are host paths already validated by the VMM. Attach extra + // volumes after the data disk and before networking, matching the + // established device order. + for (index, volume) in self.prepared.volumes.iter().enumerate() { + let id = format!("vol{index}"); + let drive = format!( + "file={},if=none,id={id},format=raw,readonly=on", + volume.source + ); + + let device = format!("virtio-blk-pci,drive={id}"); + command + .arg("-drive") + .arg(drive) + .arg("-device") + .arg(virtio_pci_device(&device, self.is_amd_sev_snp())); + } + } + fn configure_networking(&self, command: &mut Command) -> Result<()> { - let hostfwd_index = self - .prepared - .networks - .iter() - .position(|networking| networking.mode == NetworkingMode::User); + let macvtap_fds = macvtap_fd_layout(&self.prepared.networks); for (index, networking) in self.prepared.networks.iter().enumerate() { let net_id = format!("net{index}"); let mac = mac_address_for_vm_index( @@ -473,34 +645,68 @@ impl QemuCommandBuilder<'_> { &networking.mac_prefix_bytes(), index, ); - let net_device = virtio_pci_device( - &format!("virtio-net-pci,netdev={net_id},mac={mac}"), - self.is_amd_sev_snp(), - ); - let netdev = match networking.mode { + let queues = networking.queue_pairs(); + let vhost = networking.vhost_enabled(); + let mut device = format!("virtio-net-pci,netdev={net_id},mac={mac}"); + if queues > 1 { + // One vector per queue direction, plus config and control. + device.push_str(&format!(",mq=on,vectors={}", 2 * queues + 2)); + } + let net_device = virtio_pci_device(&device, self.is_amd_sev_snp()); + let netdev = match networking.nic.mode { NetworkingMode::User => { + // The user-mode backend has neither, so both are ignored + // here. A caller who *named* this mode and then asked for + // vhost or more than one queue pair is refused by the RPC; + // one who inherited it is not, and lands here. let mut netdev = format!( "user,id={net_id},net={},dhcpstart={},restrict={}", networking.net, networking.dhcp_start, if networking.restrict { "yes" } else { "no" } ); - if hostfwd_index == Some(index) { - for mapping in &self.vm.manifest.port_map { - netdev.push_str(&format!( - ",hostfwd={}:{}:{}-:{}", - mapping.protocol.as_str(), - mapping.address, - mapping.from, - mapping.to - )); + // Only the mappings that resolve to this NIC. A mapping + // lands on exactly one, and that NIC's backend decides the + // mechanism, so a bridge NIC's ports go to netd instead of + // being claimed here as well. + for mapping in &self.vm.manifest.port_map { + if ingress_nic(mapping, &self.prepared.networks) != Some(index) { + continue; } + netdev.push_str(&format!( + ",hostfwd={}:{}:{}-:{}", + mapping.protocol.as_str(), + mapping.address, + mapping.from, + mapping.to + )); } netdev } NetworkingMode::Bridge => { - tracing::info!("bridge networking: mac={mac} bridge={}", networking.bridge); - format!("bridge,id={net_id},br={}", networking.bridge) + tracing::info!( + "bridge networking: mac={mac} bridge={} vhost={vhost} queues={queues}", + networking.nic.bridge + ); + // netd owns the TAP. It is the one component here with + // CAP_NET_ADMIN, so it is the only one that can bind an + // nwfilter or create a persistent IFF_MULTI_QUEUE device -- + // and having it own every bridge TAP is what keeps a VM's + // networking from depending on which of those a node + // happens to use. + let tap = tap_name(&InterfaceIdentity { + instance_id: self.cfg.instance_id.clone(), + vm_id: self.vm.manifest.id.clone(), + nic_index: index as u32, + }); + let mut netdev = format!( + "tap,id={net_id},ifname={tap},script=no,downscript=no,vhost={}", + on_off(vhost) + ); + if queues > 1 { + netdev.push_str(&format!(",queues={queues}")); + } + netdev } NetworkingMode::Custom => { if !networking.netdev.contains(&format!("id={net_id}")) { @@ -510,6 +716,28 @@ impl QemuCommandBuilder<'_> { } networking.netdev.clone() } + NetworkingMode::Macvtap => { + if networking.device.is_empty() { + bail!("macvtap interface {index} has not been prepared by netd"); + } + let fds = macvtap_fds + .get(index) + .filter(|fds| !fds.is_empty()) + .with_context(|| { + format!("macvtap interface {index} has no launcher descriptors") + })?; + let selector = if fds.len() == 1 { + format!("fd={}", fds[0]) + } else { + let fds = fds + .iter() + .map(|fd| fd.to_string()) + .collect::<Vec<_>>() + .join(":"); + format!("fds={fds}") + }; + format!("tap,id={net_id},{selector},vhost={}", on_off(vhost)) + } }; command.arg("-netdev").arg(netdev); command.arg("-device").arg(net_device); @@ -518,10 +746,12 @@ impl QemuCommandBuilder<'_> { } fn configure_tpm_and_vsock(&self, command: &mut Command) { - if let Some(tpm_path) = self.prepared.tpm_path { + if let Some(socket) = &self.prepared.swtpm_socket { command + .arg("-chardev") + .arg(format!("socket,id=chrtpm,path={}", socket.display())) .arg("-tpmdev") - .arg(format!("passthrough,id=tpm0,path={tpm_path}")) + .arg("emulator,id=tpm0,chardev=chrtpm") .arg("-device") .arg("tpm-tis,tpmdev=tpm0"); } @@ -663,17 +893,16 @@ impl QemuCommandBuilder<'_> { if let Some(cpus) = &self.prepared.numa_cpus { arguments.splice(0..0, ["taskset", "-c", cpus].into_iter().map(String::from)); } - if !self.cfg.user.is_empty() { - arguments.splice( - 0..0, - ["sudo", "-u", &self.cfg.user].into_iter().map(String::from), - ); - } let command = arguments.remove(0); let note = serde_json::to_string(&ProcessAnnotation { kind: "cvm".to_string(), live_for: None, + // Recorded on the process rather than tracked in VMM memory, so it + // survives a VMM restart and describes the QEMU that is actually + // running. The vm-launcher wrapper copies this note verbatim, so + // TPM-backed VMs carry it too. + serial_logappend: true, })?; Ok(ProcessConfig { id: self.vm.manifest.id.clone(), @@ -706,13 +935,13 @@ impl VmConfig { } match prepared.platform { - TeePlatform::Tdx => { + CvmPlatform::Tdx => { command .arg("-machine") .arg("q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off"); self.configure_tdx_guest(command, cfg, prepared.tdx_mr_config_id.as_deref())?; } - TeePlatform::AmdSevSnp => { + CvmPlatform::AmdSevSnp => { let host_data = prepared .snp_host_data .as_deref() @@ -869,12 +1098,32 @@ mod tests { }; use super::{ - amd_sev_snp_memory_backend_arg, parse_amd_sev_snp_qmp_capabilities, virtio_pci_device, - PreparedQemuLaunch, QemuCommandBuilder, VmConfig, + amd_sev_snp_memory_backend_arg, macvtap_fd_layout, parse_amd_sev_snp_qmp_capabilities, + virtio_pci_device, PreparedQemuLaunch, PreparedVolume, QemuCommandBuilder, VmConfig, }; use crate::app::image::{Image, ImageInfo}; - use crate::app::{GpuConfig, Manifest, PortMapping, VmWorkDir}; - use crate::config::{Config, Protocol, TeePlatform, DEFAULT_CONFIG}; + use crate::app::{needs_swtpm, GpuConfig, GpuSpec, Manifest, PortMapping, VmVolume, VmWorkDir}; + use crate::config::{ + Config, CvmPlatform, NetworkFilterMode, Networking, NetworkingMode, NicNetworking, + Protocol, DEFAULT_CONFIG, + }; + use crate::netd::{tap_name, InterfaceIdentity}; + use dstack_types::{KeyProviderKind, TeeVariant}; + + #[test] + fn swtpm_is_omitted_when_simulator_provides_the_tpm() { + for platform in [TeeVariant::DstackGcpTdx, TeeVariant::DstackAwsNitroTpm] { + assert!(!needs_swtpm(KeyProviderKind::Tpm, Some(platform))); + assert!(!needs_swtpm(KeyProviderKind::Kms, Some(platform))); + } + + assert!(needs_swtpm( + KeyProviderKind::Tpm, + Some(TeeVariant::DstackTdx) + )); + assert!(needs_swtpm(KeyProviderKind::Tpm, None)); + assert!(!needs_swtpm(KeyProviderKind::Kms, None)); + } #[test] fn amd_sev_snp_memory_backend_arg_uses_passed_final_memory_size() { @@ -908,12 +1157,13 @@ mod tests { ); } - #[test] - fn qemu_command_builder_does_not_require_prepared_paths_to_exist() { + /// Minimal launch fixture. Nothing it points at has to exist on disk; every + /// test overrides the fields it asserts on. + fn test_launch_fixture() -> (Config, VmConfig, PreparedQemuLaunch) { let mut config: Config = Figment::from(Toml::string(DEFAULT_CONFIG)) .extract() .unwrap(); - config.cvm.platform = Some(TeePlatform::Tdx); + config.cvm.platform = Some(CvmPlatform::Tdx); config.cvm.qemu_path = PathBuf::from("/not-installed/qemu-system-x86_64"); config.cvm.qgs_port = None; @@ -931,6 +1181,7 @@ mod tests { protocol: Protocol::Tcp, from: 18080, to: 8080, + nic_index: None, }], created_at_ms: 0, hugepages: false, @@ -939,7 +1190,12 @@ mod tests { kms_urls: vec![], gateway_urls: vec![], no_tee: true, + simulated_tee: None, + swtpm: false, networks: vec![], + volumes: vec![VmVolume { + source: "/does-not-exist/volume.img".into(), + }], }, image: Image { info: ImageInfo { @@ -965,6 +1221,10 @@ mod tests { digest: None, tdx_measurement: None, sev_measurement: None, + gcp_measurement: None, + aws_measurement: None, + aws_pcr_replay: None, + gcp_tpm_replay: None, }, cid: 100, workdir: PathBuf::from("/does-not-exist/vm-1"), @@ -972,16 +1232,187 @@ mod tests { }; let prepared = PreparedQemuLaunch { workdir: VmWorkDir::new("/does-not-exist/vm-1"), - platform: TeePlatform::Tdx, + platform: CvmPlatform::Tdx, networks: vec![config.cvm.networking.clone(), config.cvm.networking.clone()], + volumes: vec![PreparedVolume { + source: "/does-not-exist/volume.img".into(), + }], + storage_discard: true, hugepage_numa_nodes: None, gpu_numa_nodes: HashMap::new(), numa_cpus: None, - tpm_path: None, + swtpm_socket: None, + swtpm_path: None, tdx_mr_config_id: None, snp_host_data: None, snp_launch_params: None, }; + (config, vm, prepared) + } + + /// Builds the `-netdev`/`-device` pairs for one NIC layout. + fn net_args(config: &Config, networks: Vec<Networking>) -> Vec<String> { + let (_, vm, mut prepared) = test_launch_fixture(); + prepared.networks = networks; + let process = QemuCommandBuilder { + vm: &vm, + cfg: &config.cvm, + gpus: &GpuConfig::default(), + prepared: &prepared, + } + .build() + .unwrap(); + process + .args + .windows(2) + .filter(|args| args[0] == "-netdev" || args[0] == "-device") + .map(|args| args[1].clone()) + .collect() + } + + fn bridge_network(config: &Config) -> Networking { + let mut networking = config.cvm.networking.clone(); + networking.nic.mode = NetworkingMode::Bridge; + networking.nic.bridge = "br0".into(); + // The vhost tests below are about what an opted-in node builds; the + // shipped default leaves it off. + networking.nic.vhost = Some(true); + networking + } + + #[test] + fn every_bridge_nic_gets_the_netd_tap() { + // Not only the filtered or multiqueue ones. A bridge NIC's host + // interface has one owner, so the netdev QEMU is handed does not + // change with the node's filter mode or its queue count. + let (mut config, ..) = test_launch_fixture(); + config.cvm.instance_id = "vmm-a".into(); + let mut networking = bridge_network(&config); + networking.nic.queues = Some(1); + let args = net_args(&config, vec![networking]); + let tap = tap_name(&InterfaceIdentity { + instance_id: "vmm-a".into(), + vm_id: "vm-1".into(), + nic_index: 0, + }); + assert!(args.contains(&format!( + "tap,id=net0,ifname={tap},script=no,downscript=no,vhost=on" + ))); + // A single queue pair must keep the historical device line byte for byte. + assert!(args.iter().any( + |arg| arg.starts_with("virtio-net-pci,netdev=net0,mac=") && !arg.contains("mq=on") + )); + } + + #[test] + fn disabling_vhost_keeps_the_netd_tap_and_turns_the_data_plane_off() { + let (mut config, ..) = test_launch_fixture(); + config.cvm.instance_id = "vmm-a".into(); + let mut networking = bridge_network(&config); + networking.nic.vhost = Some(false); + let args = net_args(&config, vec![networking]); + let tap = tap_name(&InterfaceIdentity { + instance_id: "vmm-a".into(), + vm_id: "vm-1".into(), + nic_index: 0, + }); + assert!(args.contains(&format!( + "tap,id=net0,ifname={tap},script=no,downscript=no,vhost=off" + ))); + } + + #[test] + fn multiqueue_bridge_uses_the_netd_tap_and_derives_vectors() { + let (mut config, ..) = test_launch_fixture(); + config.cvm.instance_id = "vmm-a".into(); + let mut networking = bridge_network(&config); + networking.nic.queues = Some(4); + let args = net_args(&config, vec![networking]); + let tap = tap_name(&InterfaceIdentity { + instance_id: "vmm-a".into(), + vm_id: "vm-1".into(), + nic_index: 0, + }); + assert!(args.contains(&format!( + "tap,id=net0,ifname={tap},script=no,downscript=no,vhost=on,queues=4" + ))); + // vectors = 2 per queue pair, plus config and control. + assert!(args.iter().any(|arg| arg.contains("mq=on,vectors=10"))); + } + + #[test] + fn macvtap_queues_take_one_inherited_descriptor_each() { + let (config, ..) = test_launch_fixture(); + let mut first = config.cvm.networking.clone(); + first.nic.mode = NetworkingMode::Macvtap; + first.nic.parent = "eth0".into(); + first.nic.vhost = Some(true); + first.device = "/dev/tap7".into(); + first.nic.queues = Some(2); + let mut second = first.clone(); + second.device = "/dev/tap9".into(); + second.nic.queues = Some(3); + + let networks = vec![first, second]; + let args = net_args(&config, networks.clone()); + assert!(args.contains(&"tap,id=net0,fds=3:4,vhost=on".to_string())); + assert!(args.contains(&"tap,id=net1,fds=5:6:7,vhost=on".to_string())); + + // The launcher must open exactly those descriptors, in that order. + let layout = macvtap_fd_layout(&networks); + assert_eq!(layout, vec![vec![3, 4], vec![5, 6, 7]]); + } + + #[test] + fn macvtap_keeps_a_single_fd_argument_for_one_queue() { + let (config, ..) = test_launch_fixture(); + let mut networking = config.cvm.networking.clone(); + networking.nic.mode = NetworkingMode::Macvtap; + networking.nic.parent = "eth0".into(); + networking.nic.vhost = Some(true); + networking.device = "/dev/tap7".into(); + let args = net_args(&config, vec![networking]); + assert!(args.contains(&"tap,id=net0,fd=3,vhost=on".to_string())); + } + + /// The operator owns a custom netdev string and the VMM cannot edit it, so + /// the generated device line must never claim more queues than that string + /// provides -- QEMU refuses the mismatch, from inside the per-VM launcher + /// where the reason is hard to see. + #[test] + fn custom_netdev_keeps_its_string_and_stays_single_queue() { + let (config, ..) = test_launch_fixture(); + let mut networking = config.cvm.networking.clone(); + networking.nic.mode = NetworkingMode::Custom; + networking.netdev = "tap,id=net0,ifname=custom0,vhost=on,queues=8".into(); + // Even a queue count that reached the entry some other way is ignored. + networking.nic.queues = Some(8); + let args = net_args(&config, vec![networking]); + assert!(args.contains(&"tap,id=net0,ifname=custom0,vhost=on,queues=8".to_string())); + assert!( + args.iter().all(|arg| !arg.contains("mq=on")), + "custom mode must not generate a multiqueue device line: {args:?}" + ); + } + + #[test] + fn user_mode_ignores_vhost_and_keeps_its_netdev() { + let (config, ..) = test_launch_fixture(); + let mut networking = config.cvm.networking.clone(); + networking.nic.mode = NetworkingMode::User; + networking.nic.vhost = Some(true); + let args = net_args(&config, vec![networking]); + assert!(args + .iter() + .any(|arg| arg.starts_with("user,id=net0,") && !arg.contains("vhost"))); + assert!(args.iter().any( + |arg| arg.starts_with("virtio-net-pci,netdev=net0,mac=") && !arg.contains("mq=on") + )); + } + + #[test] + fn qemu_command_builder_does_not_require_prepared_paths_to_exist() { + let (mut config, vm, mut prepared) = test_launch_fixture(); let process = QemuCommandBuilder { vm: &vm, @@ -1005,6 +1436,30 @@ mod tests { .args .windows(2) .any(|args| args == ["-append", "console=hvc0"])); + assert!(process.args.iter().any(|arg| { + arg == "file=/does-not-exist/vm-1/hda.img,if=none,id=hd1,discard=unmap" + })); + assert!(process.args.windows(2).any(|args| { + args == [ + "-drive", + "file=/does-not-exist/volume.img,if=none,id=vol0,format=raw,readonly=on", + ] + })); + assert!(process + .args + .iter() + .any(|arg| { arg == "virtio-blk-pci,drive=vol0" })); + let volume_position = process + .args + .iter() + .position(|arg| arg.contains("id=vol0")) + .unwrap(); + let network_position = process + .args + .iter() + .position(|arg| arg == "-netdev") + .unwrap(); + assert!(volume_position < network_position); let netdevs = process .args .windows(2) @@ -1024,5 +1479,119 @@ mod tests { .args .iter() .any(|arg| arg.contains("virtio-net-pci,netdev=net1"))); + + prepared.storage_discard = false; + let process = QemuCommandBuilder { + vm: &vm, + cfg: &config.cvm, + gpus: &GpuConfig::default(), + prepared: &prepared, + } + .build() + .unwrap(); + assert!(process.args.iter().any(|arg| { + arg == "file=/does-not-exist/vm-1/hda.img,if=none,id=hd1,discard=ignore" + })); + + for network in &mut prepared.networks { + network.nic.mode = NetworkingMode::Bridge; + network.nic.bridge = "br0".into(); + network.nic.vhost = Some(false); + } + config.cvm.instance_id = "vmm-a".into(); + config.cvm.network_filter.mode = NetworkFilterMode::Libvirt; + let process = QemuCommandBuilder { + vm: &vm, + cfg: &config.cvm, + gpus: &GpuConfig::default(), + prepared: &prepared, + } + .build() + .unwrap(); + let expected_tap = tap_name(&InterfaceIdentity { + instance_id: "vmm-a".into(), + vm_id: "vm-1".into(), + nic_index: 0, + }); + assert!(process.args.iter().any(|arg| { + arg == &format!("tap,id=net0,ifname={expected_tap},script=no,downscript=no,vhost=off") + })); + assert!(process + .args + .iter() + .all(|arg| !arg.contains("mq=on") && !arg.contains("vectors="))); + + prepared.swtpm_socket = Some(PathBuf::from("/does-not-exist/vm-1/swtpm/swtpm.sock")); + let process = QemuCommandBuilder { + vm: &vm, + cfg: &config.cvm, + gpus: &GpuConfig::default(), + prepared: &prepared, + } + .build() + .unwrap(); + assert!(process.args.windows(2).any(|args| { + args == [ + "-chardev", + "socket,id=chrtpm,path=/does-not-exist/vm-1/swtpm/swtpm.sock", + ] + })); + assert!(process + .args + .windows(2) + .any(|args| args == ["-tpmdev", "emulator,id=tpm0,chardev=chrtpm"])); + + assert!(process + .args + .iter() + .any(|arg| arg.starts_with("local,path=") && arg.contains("mount_tag=host-shared"))); + + prepared.swtpm_socket = None; + prepared.networks = vec![Networking { + nic: NicNetworking { + mode: NetworkingMode::Custom, + ..NicNetworking::default() + }, + netdev: "tap,id=wrong".into(), + ..Networking::default() + }]; + let error = QemuCommandBuilder { + vm: &vm, + cfg: &config.cvm, + gpus: &GpuConfig::default(), + prepared: &prepared, + } + .build() + .unwrap_err(); + assert!(error.to_string().contains("must contain id=net0")); + prepared.networks[0].netdev = "tap,id=net0,fd=3".into(); + QemuCommandBuilder { + vm: &vm, + cfg: &config.cvm, + gpus: &GpuConfig::default(), + prepared: &prepared, + } + .build() + .unwrap(); + + let gpu = GpuConfig { + gpus: vec![GpuSpec { + slot: "0000:02:00.0".into(), + }], + ..Default::default() + }; + let process = QemuCommandBuilder { + vm: &vm, + cfg: &config.cvm, + gpus: &gpu, + prepared: &prepared, + } + .build() + .unwrap(); + assert!(process.args.iter().any(|arg| arg == "iommufd,id=iommufd0")); + assert!(process + .args + .iter() + .any(|arg| arg.contains("vfio-pci,host=0000:02:00.0"))); } } diff --git a/dstack/vmm/src/app/registry.rs b/dstack/vmm/src/app/registry.rs index d329af7c5..dbd763f61 100644 --- a/dstack/vmm/src/app/registry.rs +++ b/dstack/vmm/src/app/registry.rs @@ -5,12 +5,13 @@ //! OCI Distribution API client for pulling dstack guest images directly from //! a container registry without requiring a local Docker daemon. -use std::path::Path; +use std::{io::Read, path::Path}; use anyhow::{bail, Context, Result}; use flate2::read::GzDecoder; use reqwest::Client; use serde::Deserialize; +use sha2::{Digest, Sha256}; use tracing::info; fn build_client() -> Result<Client> { @@ -91,6 +92,7 @@ async fn list_tags_with_token(client: &Client, registry: &str, repo: &str) -> Re /// Fetches the OCI manifest, downloads each layer blob, and extracts /// the tar (gzipped) contents into a flat directory. pub async fn pull_and_extract(image_ref: &str, tag: &str, image_path: &Path) -> Result<()> { + validate_registry_tag(tag)?; let (registry, repo) = parse_image_ref(image_ref)?; let client = build_client()?; @@ -250,6 +252,22 @@ async fn download_and_extract_layers( } let bytes = response.bytes().await.context("failed to read blob body")?; + if bytes.len() as u64 != layer.size { + bail!( + "blob {} size mismatch: expected {}, received {}", + layer.digest, + layer.size, + bytes.len() + ); + } + let actual_digest = format!("sha256:{:x}", Sha256::digest(&bytes)); + if actual_digest != layer.digest { + bail!( + "blob digest mismatch: expected {}, received {}", + layer.digest, + actual_digest + ); + } extract_layer(&bytes, &layer.media_type, dest)?; } @@ -265,14 +283,10 @@ fn extract_layer(data: &[u8], media_type: &str, dest: &Path) -> Result<()> { if is_gzip { let decoder = GzDecoder::new(data); let mut archive = tar::Archive::new(decoder); - archive - .unpack(dest) - .context("failed to extract gzipped tar layer")?; + unpack_archive(&mut archive, dest).context("failed to extract gzipped tar layer")?; } else { let mut archive = tar::Archive::new(data); - archive - .unpack(dest) - .context("failed to extract tar layer")?; + unpack_archive(&mut archive, dest).context("failed to extract tar layer")?; } // Remove docker/OCI artifact directories that may appear in layers @@ -286,6 +300,19 @@ fn extract_layer(data: &[u8], media_type: &str, dest: &Path) -> Result<()> { Ok(()) } +fn unpack_archive<R: Read>(archive: &mut tar::Archive<R>, dest: &Path) -> Result<()> { + for entry in archive.entries().context("failed to read tar entries")? { + let mut entry = entry.context("failed to read tar entry")?; + if !entry + .unpack_in(dest) + .context("failed to unpack tar entry")? + { + bail!("archive entry escapes destination"); + } + } + Ok(()) +} + // ─── Token auth ───────────────────────────────────────────────────────────── /// Try to fetch a Bearer token. Returns None if the registry doesn't need one. @@ -353,6 +380,21 @@ fn parse_www_authenticate(header: &str) -> (String, String) { // ─── Helpers ──────────────────────────────────────────────────────────────── +fn validate_registry_tag(tag: &str) -> Result<()> { + if tag.is_empty() + || tag == "." + || tag == ".." + || tag.contains('/') + || tag.contains('\\') + || !tag + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-')) + { + bail!("invalid registry tag"); + } + Ok(()) +} + fn determine_output_dir(tag: &str, image_path: &Path) -> std::path::PathBuf { let dir_name = if tag.starts_with("dstack-") { tag.to_string() @@ -441,6 +483,26 @@ struct OciIndexEntry { mod tests { use super::*; + #[test] + fn registry_tag_is_confined_to_one_image_store_entry() { + for valid in ["v0.6.0", "dstack-fixture_1", "sha256-deadbeef"] { + validate_registry_tag(valid).unwrap(); + } + for invalid in [ + "", + ".", + "..", + "../escape", + "dstack-../../escape", + "nested/tag", + r"nested\tag", + "tag,option", + "tag=value", + ] { + assert!(validate_registry_tag(invalid).is_err(), "{invalid}"); + } + } + #[test] fn test_parse_image_ref_private_registry() { let (reg, repo) = parse_image_ref("cr.kvin.wang/dstack/guest-image").unwrap(); diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index d039418c8..48af6f864 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -11,16 +11,16 @@ use dstack_vmm_rpc as pb; use fs_err as fs; use supervisor_client::supervisor::ProcessInfo; -use super::{ - network::{mac_address_for_vm_index, resolved_networks}, - Manifest, VmState, VmWorkDir, -}; -use crate::config::{CvmConfig, GatewayConfig, Networking, NetworkingMode}; +use super::{network::mac_address_for_vm_index, Manifest, VmState, VmWorkDir}; +use crate::config::{GatewayConfig, Networking, NetworkingMode, NicNetworking}; pub(crate) struct VmInfo { pub manifest: Manifest, pub workdir: PathBuf, pub status: &'static str, + /// Whether a QEMU process exists for this VM right now. The NICs it built + /// are real only while it does. + pub running: bool, pub uptime: String, pub exited_at: Option<String>, pub instance_id: Option<String>, @@ -33,30 +33,92 @@ pub(crate) struct VmInfo { pub runtime_networks: Vec<Networking>, } -fn networking_mode_name(mode: NetworkingMode) -> &'static str { - match mode { - NetworkingMode::Bridge => "bridge", - NetworkingMode::User => "user", - NetworkingMode::Custom => "custom", - } -} - fn networking_backend_name(mode: NetworkingMode) -> &'static str { match mode { NetworkingMode::Bridge => "tap_bridge", NetworkingMode::User => "slirp", NetworkingMode::Custom => "custom", + NetworkingMode::Macvtap => "macvtap", } } -fn networking_to_proto(networking: &Networking) -> pb::NetworkingConfig { +/// The resolved NICs a launch built, or would build, as the RPC reports them. +fn interfaces_to_proto( + vm_id: &str, + effective_networks: &[Networking], +) -> Vec<pb::NetworkInterfaceStatus> { + effective_networks + .iter() + .enumerate() + .map(|(index, networking)| { + let mac = mac_address_for_vm_index(vm_id, &networking.mac_prefix_bytes(), index); + pb::NetworkInterfaceStatus { + mode: networking.nic.mode.as_str().into(), + backend: networking_backend_name(networking.nic.mode).into(), + mac, + bridge_name: (networking.nic.mode == NetworkingMode::Bridge) + .then(|| networking.nic.bridge.clone()), + netdev_id: Some(format!("net{index}")), + // Custom mode hands the operator the whole netdev string and the + // VMM never parses it, so it has no data-plane state to report. + // Reporting the resolved fields anyway would assert "vhost: off, + // queues: 1" over a netdev the operator may have written with + // `vhost=on,queues=8`. + // + // Otherwise: settled at launch, so an entry carrying no decision + // was written before this VMM recorded one -- by a build that + // had no vhost at all, which is what it should read as. + // Recomputing here instead would let an edit to node + // configuration change what a running VM is said to use. + vhost: (networking.nic.mode != NetworkingMode::Custom) + .then(|| networking.nic.vhost.is_some() && networking.vhost_enabled()), + queues: (networking.nic.mode != NetworkingMode::Custom) + .then(|| networking.queue_pairs()), + // Node-decided, so it belongs with the rest of the resolved + // state. The VM's own record cannot carry one. + macvtap_mode: (networking.nic.mode == NetworkingMode::Macvtap) + .then(|| networking.macvtap_mode.clone()), + } + }) + .collect() +} + +pub(crate) fn networking_to_proto(networking: &NicNetworking) -> pb::NetworkingConfig { + // An entry that inherited its backend reports no mode, so it must report + // none of the fields that only make sense alongside one: a mode-less + // override carrying, say, a parent is something the deployment RPC + // rejects, which would strand the VM's tuning as uneditable. + let pins_backend = !networking.inherit_mode; pb::NetworkingConfig { - mode: networking_mode_name(networking.mode).into(), - bridge_name: if networking.mode == NetworkingMode::Bridge { + // An entry that only tuned the data plane named no backend, and the + // deployment RPC spells that as an empty mode. Reporting the node's + // current mode here would turn a read-modify-write into a request to + // pin it -- which policy may not even permit the caller to make. + mode: if networking.inherit_mode { + String::new() + } else { + networking.mode.as_str().into() + }, + bridge_name: if pins_backend && networking.mode == NetworkingMode::Bridge { networking.bridge.clone() } else { String::new() }, + // Scope the macvtap fields to macvtap, the way bridge_name is scoped to + // bridge. Reporting an inherited parent on a bridge NIC produced a + // configuration that could be read but not sent back: the deployment + // RPC rejects `parent` outside macvtap mode. + parent: if pins_backend && networking.mode == NetworkingMode::Macvtap { + networking.parent.clone() + } else { + String::new() + }, + // The forwarding mode is node-controlled, so a VM never pins one and + // the type it stores can no longer carry one. It stays on the wire + // because the deployment RPC still has to reject a caller that sets it. + macvtap_mode: String::new(), + vhost: networking.vhost, + queues: networking.queues, } } @@ -65,15 +127,20 @@ fn sanitize_optional<T: AsRef<str>>(value: Option<T>) -> Option<T> { } impl VmInfo { - pub fn effective_networks(&self, cvm: &CvmConfig) -> Vec<Networking> { - if self.runtime_networks.is_empty() { - resolved_networks(&self.manifest, cvm) - } else { - self.runtime_networks.clone() - } - } - - pub fn to_pb(&self, gateway: &GatewayConfig, cvm: &CvmConfig, brief: bool) -> pb::VmInfo { + /// Takes no `CvmConfig` on purpose. Everything it reports about a VM's + /// data plane was decided when that VM launched and written into + /// `effective_networks`; consulting node configuration here is what let an + /// operator's edit change what a running VM was said to be using. + /// + /// `effective_networks` is passed in rather than derived for the same + /// reason, plus one more: a stopped VM's NICs are a prediction, and only + /// the caller can consult netd to make the prediction its launch would. + pub fn to_pb( + &self, + gateway: &GatewayConfig, + brief: bool, + effective_networks: &[Networking], + ) -> pb::VmInfo { let workdir = VmWorkDir::new(&self.workdir); let vm_config = workdir.manifest(); let custom_gateway_urls = vm_config @@ -87,30 +154,17 @@ impl VmInfo { .map(networking_to_proto) .collect::<Vec<_>>(); let configured_networking = configured_networks.first().cloned(); - let interfaces = self - .effective_networks(cvm) - .iter() - .enumerate() - .map(|(index, networking)| { - let mac = mac_address_for_vm_index( - &self.manifest.id, - &networking.mac_prefix_bytes(), - index, - ); - pb::NetworkInterfaceStatus { - mode: networking_mode_name(networking.mode).into(), - backend: networking_backend_name(networking.mode).into(), - mac, - bridge_name: (networking.mode == NetworkingMode::Bridge) - .then(|| networking.bridge.clone()), - netdev_id: Some(format!("net{index}")), - } - }) - .collect(); + let interfaces = interfaces_to_proto(&self.manifest.id, effective_networks); pb::VmInfo { id: self.manifest.id.clone(), name: self.manifest.name.clone(), status: self.status.into(), + // The one predicate that says whether `interfaces` above is what a + // process built or what the next launch would build. Clients used to + // re-derive it from `status`, which answers a different question: + // a VM being removed with QEMU still up is not "running" by that + // string, yet its NICs are real. + running: self.running, uptime: self.uptime.clone(), boot_progress: self.boot_progress.clone(), boot_error: self.boot_error.clone(), @@ -144,6 +198,7 @@ impl VmInfo { .port_map .iter() .map(|mapping| pb::PortMapping { + nic_index: mapping.nic_index.map(|index| index as u32), protocol: mapping.protocol.as_str().into(), host_address: mapping.address.to_string(), host_port: mapping.from as u32, @@ -167,6 +222,10 @@ impl VmInfo { gateway_urls: custom_gateway_urls.clone(), stopped, no_tee, + simulated_tee: self + .manifest + .simulated_tee + .map(|platform| platform.as_str().to_string()), networking: configured_networking, networks: configured_networks, }) @@ -253,6 +312,7 @@ impl VmState { workdir: workdir.path().to_path_buf(), instance_id, status, + running: is_running, uptime, exited_at: Some(exited_at), boot_progress: self.state.boot_progress.clone(), @@ -268,7 +328,68 @@ impl VmState { #[cfg(test)] mod tests { - use super::sanitize_optional; + use super::{interfaces_to_proto, networking_to_proto, sanitize_optional}; + use crate::config::{NetworkingMode, NicNetworking}; + + /// Custom mode hands the operator the whole netdev string and the VMM never + /// parses it, so it has no data-plane state to report. Reporting the + /// resolved defaults instead asserted "vhost off, one queue" over a netdev + /// the operator may well have written as `vhost=on,queues=8`. + #[test] + fn a_custom_netdev_reports_no_data_plane_rather_than_the_wrong_one() { + use crate::config::Networking; + + let custom = Networking { + nic: NicNetworking { + mode: NetworkingMode::Custom, + ..NicNetworking::default() + }, + netdev: "tap,id=net0,ifname=custom0,vhost=on,queues=8".into(), + ..Networking::default() + }; + let interfaces = interfaces_to_proto("vm-1", &[custom]); + assert_eq!(interfaces[0].backend, "custom"); + assert_eq!(interfaces[0].vhost, None); + assert_eq!(interfaces[0].queues, None); + + // Every other backend still answers the question. + let bridge = Networking { + nic: NicNetworking { + mode: NetworkingMode::Bridge, + vhost: Some(true), + queues: Some(4), + ..NicNetworking::default() + }, + ..Networking::default() + }; + let interfaces = interfaces_to_proto("vm-1", &[bridge]); + assert_eq!(interfaces[0].vhost, Some(true)); + assert_eq!(interfaces[0].queues, Some(4)); + } + + #[test] + fn a_reported_interface_can_be_sent_back_unchanged() { + // GetInfo output feeds UpdateVm, so anything it reports has to satisfy + // the deployment RPC's own validation. What a VM stores can no longer + // carry a node-owned field at all -- `parent` here belongs to bridge + // mode's own entry only because the type still allows both backends' + // identity fields, and reporting still scopes it to the owning mode. + let networking = NicNetworking { + mode: NetworkingMode::Bridge, + bridge: "br0".into(), + parent: "eth0".into(), + vhost: Some(false), + queues: Some(2), + ..NicNetworking::default() + }; + let proto = networking_to_proto(&networking); + assert_eq!(proto.mode, "bridge"); + assert_eq!(proto.bridge_name, "br0"); + assert!(proto.parent.is_empty()); + assert!(proto.macvtap_mode.is_empty()); + assert_eq!(proto.vhost, Some(false)); + assert_eq!(proto.queues, Some(2)); + } #[test] fn sanitize_optional_filters_empty_owned_values() { diff --git a/dstack/vmm/src/app/workdir.rs b/dstack/vmm/src/app/workdir.rs index 6984cd4b5..a9822c7ec 100644 --- a/dstack/vmm/src/app/workdir.rs +++ b/dstack/vmm/src/app/workdir.rs @@ -104,6 +104,18 @@ impl VmWorkDir { self.workdir.join("shared") } + pub fn swtpm_state_dir(&self) -> PathBuf { + self.workdir.join("swtpm") + } + + pub fn swtpm_socket(&self) -> PathBuf { + self.swtpm_state_dir().join("swtpm.sock") + } + + pub fn launch_spec_path(&self) -> PathBuf { + self.workdir.join("launch.json") + } + pub fn app_compose_path(&self) -> PathBuf { self.shared_dir().join(APP_COMPOSE) } @@ -127,25 +139,10 @@ impl VmWorkDir { self.shared_dir().join(INSTANCE_INFO) } - pub fn guest_ip_path(&self) -> PathBuf { - self.workdir.join("guest-ip") - } - pub fn runtime_networks_path(&self) -> PathBuf { self.workdir.join("runtime-networks.json") } - pub fn guest_ip(&self) -> Option<String> { - fs::read_to_string(self.guest_ip_path()) - .ok() - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - } - - pub fn set_guest_ip(&self, ip: &str) -> Result<()> { - fs::write(self.guest_ip_path(), ip).context("failed to write guest IP") - } - pub fn runtime_networks(&self) -> Vec<Networking> { fs::read_to_string(self.runtime_networks_path()) .ok() @@ -154,7 +151,14 @@ impl VmWorkDir { } pub fn set_runtime_networks(&self, networks: &[Networking]) -> Result<()> { - let serialized = serde_json::to_vec(networks)?; + // A macvtap device path is valid only while its host interface exists. + // Keep it in memory for launch preparation, but never persist it across + // VMM restarts where the same /dev/tapN may identify another device. + let mut persistent_networks = networks.to_vec(); + for network in &mut persistent_networks { + network.device.clear(); + } + let serialized = serde_json::to_vec(&persistent_networks)?; safe_write::safe_write(self.runtime_networks_path(), serialized) .context("failed to write runtime networks") } @@ -171,10 +175,6 @@ impl VmWorkDir { self.workdir.join("serial.log") } - pub fn serial_history_file(&self) -> PathBuf { - self.workdir.join("serial.history.log") - } - pub fn serial_pty(&self) -> PathBuf { self.workdir.join("serial.pty") } @@ -261,6 +261,7 @@ mod tests { use fs_err as fs; use super::VmWorkDir; + use crate::config::Networking; #[test] fn runtime_networks_snapshot_replaces_target_instead_of_following_it() -> Result<()> { @@ -282,4 +283,30 @@ mod tests { fs::remove_dir_all(temp)?; Ok(()) } + + #[test] + fn runtime_networks_snapshot_omits_ephemeral_device_paths() -> Result<()> { + let temp = std::env::temp_dir().join(format!( + "dstack-vmm-runtime-networks-device-test-{}", + SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos() + )); + let workdir = VmWorkDir::new(temp.join("vm")); + fs::create_dir_all(workdir.path())?; + let network: Networking = serde_json::from_value(serde_json::json!({ + "mode": "macvtap", + "parent": "br0", + "macvtap_mode": "private", + "device": "/dev/tap42" + }))?; + + workdir.set_runtime_networks(&[network])?; + + let persisted = workdir.runtime_networks(); + assert_eq!(persisted.len(), 1); + assert_eq!(persisted[0].nic.parent, "br0"); + assert!(persisted[0].device.is_empty()); + assert!(!fs::read_to_string(workdir.runtime_networks_path())?.contains("/dev/tap42")); + fs::remove_dir_all(temp)?; + Ok(()) + } } diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 330a2e440..49f013f57 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::{net::IpAddr, path::PathBuf, process::Command, str::FromStr}; +use std::{collections::BTreeMap, net::IpAddr, path::PathBuf, process::Command, str::FromStr}; use anyhow::{bail, Context, Result}; use load_config::load_config; @@ -20,59 +20,61 @@ fn detect_qemu_version(qemu_path: &PathBuf) -> Result<String> { let output = Command::new(qemu_path) .arg("--version") .output() - .context("Failed to execute qemu --version")?; + .context("failed to execute qemu --version")?; if !output.status.success() { - bail!("QEMU version command failed with status: {}", output.status); + bail!("qemu --version failed with status: {}", output.status); } - let version_output = - String::from_utf8(output.stdout).context("QEMU version output is not valid UTF-8")?; - - parse_qemu_version_from_output(&version_output) - .context("Could not parse QEMU version from output") + // A wrapper script can print a banner before the version line, on either + // stream, so search both instead of just the first line of stdout. + let combined = format!( + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + parse_qemu_version_from_output(&combined).with_context(|| { + format!( + "unrecognized qemu --version output: {}", + first_lines(&combined) + ) + }) } -fn parse_qemu_version_from_output(output: &str) -> Result<String> { - // Parse version from output like: - // "QEMU emulator version 8.2.2 (Debian 2:8.2.2+ds-0ubuntu1.4+tdx1.0)" - // "QEMU emulator version 9.1.0" - let version = output +/// A bounded excerpt of a command's output, for error messages. +fn first_lines(output: &str) -> String { + output .lines() - .next() - .and_then(|line| { - let words: Vec<&str> = line.split_whitespace().collect(); - - // First try: Look for "version" keyword and get the next word (only if it looks like a version) - if let Some(version_idx) = words.iter().position(|&word| word == "version") { - if let Some(next_word) = words.get(version_idx + 1) { - // Only use the word after "version" if it looks like a version number - if next_word.chars().next().is_some_and(|c| c.is_ascii_digit()) - && (next_word.contains('.') - || next_word.chars().all(|c| c.is_ascii_digit() || c == '-')) - { - return Some(*next_word); - } - } - } + .filter(|line| !line.trim().is_empty()) + .take(2) + .collect::<Vec<_>>() + .join(" | ") + .chars() + .take(200) + .collect() +} - // Fallback: find first word that looks like a version number - words - .iter() - .find(|word| { - // Check if word starts with digit and contains dots (version-like) - word.chars().next().is_some_and(|c| c.is_ascii_digit()) - && (word.contains('.') - || word.chars().all(|c| c.is_ascii_digit() || c == '-')) - }) - .copied() - }) - .context("Could not parse QEMU version from output")?; +fn looks_like_version(word: &str) -> bool { + word.chars().next().is_some_and(|c| c.is_ascii_digit()) + && (word.contains('.') || word.chars().all(|c| c.is_ascii_digit() || c == '-')) +} - // Extract just the version number (e.g., "8.2.2" from "8.2.2+ds-0ubuntu1.4+tdx1.0") - let clean_version = version.split('+').next().unwrap_or(version).to_string(); +/// QEMU's own wording: `QEMU emulator version 9.2.1 (Debian ...)`. +fn version_after_qemu_marker(line: &str) -> Option<&str> { + let rest = line.split_once("QEMU emulator version ")?.1; + let word = rest.split_whitespace().next()?; + looks_like_version(word).then_some(word) +} - Ok(clean_version) +fn parse_qemu_version_from_output(output: &str) -> Result<String> { + // Only QEMU's own wording counts, wherever it appears. Guessing at a + // version-shaped word instead would read one out of a wrapper's banner. + let version = output + .lines() + .find_map(version_after_qemu_marker) + .context("no `QEMU emulator version` line")?; + // "8.2.2+ds-0ubuntu1.4+tdx1.0" -> "8.2.2" + Ok(version.split('+').next().unwrap_or(version).to_string()) } pub fn load_config_figment(config_file: Option<&str>) -> Figment { @@ -109,12 +111,12 @@ impl Protocol { #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] -pub enum TeePlatform { +pub enum CvmPlatform { Tdx, AmdSevSnp, } -impl TeePlatform { +impl CvmPlatform { /// Detect the host TEE platform from /proc/cpuinfo. Used when the operator /// did not pin a platform in the config (`platform` omitted, or `auto`). pub fn detect() -> Self { @@ -166,7 +168,55 @@ pub struct PortMappingConfig { #[derive(Debug, Clone, Deserialize)] pub struct AutoRestartConfig { pub enabled: bool, + /// How often the supervisor state is sampled. pub interval: u64, + /// Maximum consecutive automatic restart attempts before intervention. + pub max_retries: u32, + /// Delay before the first retry. Later retries use exponential backoff. + pub initial_backoff: u64, + /// Upper bound for the exponential retry delay. + pub max_backoff: u64, + /// Continuous healthy runtime required to reset the retry budget. + pub reset_window: u64, +} + +/// Retention for the logs a CVM writes into its work directory. +/// +/// Currently governs serial.log. The caps are deliberately not named after it, +/// because rotation itself is generic (see `crate::logrotate`) and stdout/stderr +/// are the obvious next call sites. +#[derive(Debug, Clone, Deserialize)] +pub struct LogConfig { + /// Max size of a live log. QEMU appends to serial.log for the whole life of + /// a boot, so without a cap a chatty guest can fill the host disk. Past + /// this size the log is rotated and truncated in place. 0 disables + /// rotation. + #[serde(with = "size_parser::human_size")] + pub max_bytes: u64, + + /// Rotated segments to keep. Follows logrotate semantics: the oldest is + /// discarded. + pub max_backups: usize, + + /// How often a live log is checked against `max_bytes`, in seconds. + pub check_interval_secs: u64, +} + +impl AutoRestartConfig { + pub fn validate(&self) -> Result<()> { + if self.enabled { + if self.interval == 0 { + bail!("cvm.auto_restart.interval must be greater than zero when enabled"); + } + if self.initial_backoff == 0 { + bail!("cvm.auto_restart.initial_backoff must be greater than zero when enabled"); + } + if self.initial_backoff > self.max_backoff { + bail!("cvm.auto_restart.initial_backoff must not exceed max_backoff"); + } + } + Ok(()) + } } impl PortMappingConfig { @@ -181,7 +231,7 @@ impl PortMappingConfig { /// Deserialize the optional `platform` config field. `None` (field omitted, or /// the legacy literal `auto`) means "detect the host TEE"; `tdx` / `amd-sev-snp` /// pin a platform. Keeping `auto` accepted preserves existing vmm.toml configs. -fn deserialize_platform<'de, D>(deserializer: D) -> Result<Option<TeePlatform>, D::Error> +fn deserialize_platform<'de, D>(deserializer: D) -> Result<Option<CvmPlatform>, D::Error> where D: serde::Deserializer<'de>, { @@ -195,8 +245,8 @@ where Ok( match Option::<PlatformSetting>::deserialize(deserializer)? { None | Some(PlatformSetting::Auto) => None, - Some(PlatformSetting::Tdx) => Some(TeePlatform::Tdx), - Some(PlatformSetting::AmdSevSnp) => Some(TeePlatform::AmdSevSnp), + Some(PlatformSetting::Tdx) => Some(CvmPlatform::Tdx), + Some(PlatformSetting::AmdSevSnp) => Some(CvmPlatform::AmdSevSnp), }, ) } @@ -204,8 +254,30 @@ where impl CvmConfig { /// The effective TEE platform: the configured one, or host auto-detection /// when left unset (`platform` omitted / `auto`). - pub fn resolved_platform(&self) -> TeePlatform { - self.platform.unwrap_or_else(TeePlatform::detect) + pub fn resolved_platform(&self) -> CvmPlatform { + self.platform.unwrap_or_else(CvmPlatform::detect) + } + + /// The QEMU version to declare for a VM being started now. + /// + /// The verifier picks an ACPI table model from this, so it has to + /// describe the binary this start executes -- not one detected earlier + /// and cached across a package upgrade. An explicit config value wins; + /// otherwise the binary at `qemu_path` is asked. Neither available fails + /// the start, rather than booting a CVM that cannot be attested. + pub fn resolve_qemu_version(&self) -> Result<String> { + if let Some(version) = &self.qemu_version { + return Ok(version.clone()); + } + let version = detect_qemu_version(&self.qemu_path).with_context(|| { + format!( + "failed to detect the QEMU version of {}; \ + set `qemu_version` in vmm.toml to declare it explicitly", + self.qemu_path.display() + ) + })?; + info!("QEMU version: {version}"); + Ok(version) } } @@ -224,20 +296,31 @@ pub enum TdxAttestationVariantConfig { } impl TdxAttestationVariantConfig { - const TWO_GIB_MIB: u32 = 2 * 1024; - const THREE_GIB_MIB: u32 = 3 * 1024; - - pub fn resolve(self, memory_mib: u32, image_supports_lite: bool) -> TdxAttestationVariant { + /// `auto` follows the image and nothing else. + /// + /// It used to also refuse lite below 3 GiB, exempting exactly 2 GiB, + /// because QEMU's setup-header rewrite moves the initrd with guest RAM and + /// so makes the patched kernel Authenticode hash memory-dependent. Images + /// whose OVMF normalizes the setup header measure the shipped `bzImage` + /// instead, which no guest RAM size can move, and that is every image the + /// build system can now produce: it hardcodes `kernel_header_normalized` + /// and fails when the OVMF patch does not apply. + /// + /// A pre-normalization image is the one case this no longer covers. The + /// no-download verifier still rejects those below the threshold, with an + /// error naming the memory sizes and telling the operator to re-emit the + /// image, so the failure is explicit rather than a silent mismatch. Set + /// `tdx_attestation_variant = "legacy"` to keep running one as is. + pub fn resolve(self, image_supports_lite: bool) -> TdxAttestationVariant { + use TdxAttestationVariant::{Legacy, Lite}; match self { - Self::Legacy => TdxAttestationVariant::Legacy, - Self::Lite => TdxAttestationVariant::Lite, + Self::Legacy => Legacy, + Self::Lite => Lite, Self::Auto => { - if memory_mib < Self::THREE_GIB_MIB && memory_mib != Self::TWO_GIB_MIB { - TdxAttestationVariant::Legacy - } else if image_supports_lite { - TdxAttestationVariant::Lite + if image_supports_lite { + Lite } else { - TdxAttestationVariant::Legacy + Legacy } } } @@ -250,16 +333,33 @@ pub struct CvmConfig { /// the host TEE from /proc/cpuinfo (AMD SEV-SNP vs Intel TDX); set `tdx` or /// `amd-sev-snp` to force a platform. #[serde(default, deserialize_with = "deserialize_platform")] - pub platform: Option<TeePlatform>, + pub platform: Option<CvmPlatform>, pub qemu_path: PathBuf, /// The URL of the KMS server pub kms_urls: Vec<String>, + /// Randomize KMS failover order independently for each CVM. + #[serde(default)] + pub shuffle_kms_urls: bool, /// The URL of the dstack-gateway server #[serde(alias = "tproxy_urls")] pub gateway_urls: Vec<String>, + /// Independently operated gateway clusters. URLs in each entry are + /// failover endpoints for that cluster. + #[serde(default)] + pub gateway_clusters: Vec<dstack_types::GatewayClusterConfig>, + /// Randomize gateway failover order independently for each CVM. + #[serde(default)] + pub shuffle_gateway_urls: bool, /// The URL of the PCCS server #[serde(default)] pub pccs_url: String, + /// Node-local credentials and collateral settings used when an individual + /// VM requests a simulated TEE platform. + #[serde(default)] + pub tee_simulator: Option<dstack_types::TeeSimulatorConfig>, + /// Optional NVIDIA OCSP/RIM cache passed to guests in sys-config. + #[serde(default)] + pub nvidia_attestation_proxy_url: Option<String>, /// The URL of the Docker registry pub docker_registry: String, /// The start of the CID pool that allocates CIDs to VMs @@ -275,9 +375,6 @@ pub struct CvmConfig { pub qmp_socket: bool, /// GPU configuration pub gpu: GpuConfig, - /// Use sudo to run the VM - pub user: String, - /// Auto restart configuration pub auto_restart: AutoRestartConfig, @@ -295,18 +392,49 @@ pub struct CvmConfig { pub qemu_pci_hole64_size: u64, /// QEMU hotplug_off pub qemu_hotplug_off: bool, - /// TDX attestation/hash scheme policy. `legacy` keeps the existing - /// digest.txt + dstack-acpi-tables verifier path; `lite` opts into the - /// split measurement CBOR + no-QEMU verifier path; `auto` selects `legacy` for - /// CVMs below 3 GiB except exactly 2 GiB, otherwise uses `lite` when the - /// image carries TDX measurement material and falls back to `legacy`. + /// digest.txt measurement path; `lite` opts into split measurement CBOR; + /// `auto` uses `lite` when the image carries TDX measurement material and + /// falls back to `legacy` when it does not. See + /// [`TdxAttestationVariantConfig::resolve`] for why memory size no longer + /// takes part. #[serde(default)] pub tdx_attestation_variant: TdxAttestationVariantConfig, /// Networking configuration pub networking: Networking, + /// Network backends that deployment RPC callers may request explicitly. + /// The node default is still used when a request omits networking. + #[serde(default = "default_allowed_network_modes")] + pub allowed_network_modes: Vec<NetworkingMode>, + + /// Host bridges that deployment RPC callers may select explicitly. + /// An empty list permits only the node's default bridge. + #[serde(default)] + pub allowed_bridges: Vec<String>, + + /// Host interfaces that deployment RPC callers may select explicitly as + /// macvtap parents. An empty list permits only the node default parent. + #[serde(default)] + pub allowed_macvtap_parents: Vec<String>, + + /// Largest virtio-net queue pair count a deployment RPC caller may request. + /// There is no node-wide count for it to bind; lowering it below the + /// scaling cap does lower the vCPU-scaled default too. + #[serde(default = "default_max_net_queues")] + pub max_net_queues: u32, + + /// Optional host-side filtering for bridge interfaces. This filter does + /// not apply to macvtap interfaces. + #[serde(default)] + pub network_filter: NetworkFilterConfig, + + /// Stable namespace for TAP names when several VMMs share one host. + /// An empty value is derived from the absolute run directory. + #[serde(default)] + pub instance_id: String, + /// Host sharing mode. (9p, vhd, vvfat) pub host_share_mode: String, @@ -320,12 +448,14 @@ pub struct CvmConfig { #[serde(default)] pub product: ProductConfig, - /// Max size in bytes for serial.history.log (default 4MB). - /// Previous boot serial logs are appended here before each restart. - /// Accepts human-readable sizes like "4MB", "512KB". - #[serde(default = "default_serial_history_max_bytes")] - #[serde(with = "size_parser::human_size")] - pub serial_history_max_bytes: u64, + /// Guest log retention. + pub log: LogConfig, + + /// Directory holding attachable volume images (e.g. pre-baked verity + /// volumes). A deploy may only attach files under this directory, referenced + /// by bare file name. Empty (the default) disables volume attachment. + #[serde(default)] + pub volumes_dir: String, } /// SMBIOS product information configuration. @@ -373,6 +503,10 @@ pub struct GpuConfig { pub include: Vec<String>, /// Allow attach all GPUs pub allow_attach_all: bool, + /// Reset each GPU's dedicated upstream PCIe bus before QEMU attaches it. + pub sanitize_on_attach: bool, + /// Shared deadline for all GPUs to become VFIO-ready after SBR. + pub sbr_timeout_ms: u64, } impl GpuConfig { @@ -406,6 +540,9 @@ pub struct AuthConfig { pub enabled: bool, /// The API tokens pub tokens: Vec<String>, + /// Optional Apache htpasswd file for HTTP Basic authentication. + #[serde(default)] + pub htpasswd_file: PathBuf, } #[derive(Debug, Clone, Default, Deserialize)] @@ -458,6 +595,10 @@ pub struct Config { /// CVM configuration pub cvm: CvmConfig, + + /// Privileged host networking service configuration. + #[serde(default)] + pub netd: NetdConfig, /// Gateway configuration pub gateway: GatewayConfig, @@ -474,12 +615,124 @@ pub struct Config { pub key_provider: KeyProviderConfig, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum NetworkFilterMode { + #[default] + None, + Libvirt, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct NetworkFilterConfig { + #[serde(default)] + pub mode: NetworkFilterMode, + #[serde(default = "default_libvirt_filter")] + pub filter: String, + #[serde(default)] + pub parameters: BTreeMap<String, String>, +} + +impl NetworkFilterConfig { + /// Whether every bridge TAP on this node must carry an nwfilter binding. + pub fn requires_binding(&self) -> bool { + self.mode == NetworkFilterMode::Libvirt + } +} + +impl Default for NetworkFilterConfig { + fn default() -> Self { + Self { + mode: NetworkFilterMode::None, + filter: default_libvirt_filter(), + parameters: BTreeMap::new(), + } + } +} + +fn default_libvirt_filter() -> String { + "clean-traffic".to_string() +} + +#[derive(Debug, Clone, Deserialize)] +pub struct NetdConfig { + #[serde(default = "default_netd_socket")] + pub socket: PathBuf, + /// Filesystem permissions applied when netd creates its own socket. + /// Systemd-activated sockets use `SocketMode` from the socket unit. + pub socket_mode: u32, + #[serde(default = "default_libvirt_uri")] + pub libvirt_uri: String, + /// The bridge filtering policy netd enforces and applies: whether a binding + /// is required, which nwfilter it names, and with what parameters. + /// + /// netd holds this itself rather than taking it from each request. It is + /// the privileged side of the socket, and a caller that chose the filter + /// could name one that drops nothing -- `allow-arp` has no drop rule at all + /// -- or pin `clean-traffic` to the gateway's MAC and IP through its + /// parameters, and still satisfy a policy that only asked for "some + /// filter". + /// + /// Unset derives it from `cvm.network_filter` in the same file, which is + /// the whole answer whenever netd and the VMM share one `vmm.toml` -- the + /// normal deployment. Set it explicitly when netd runs with a config that + /// carries no `[cvm]` section, so the daemon holding the privilege is never + /// left inferring policy from a file that does not state it. + #[serde(default)] + pub network_filter: Option<NetworkFilterConfig>, +} + +impl Default for NetdConfig { + fn default() -> Self { + Self { + socket: default_netd_socket(), + socket_mode: 0o660, + libvirt_uri: default_libvirt_uri(), + network_filter: None, + } + } +} + +impl NetdConfig { + /// Resolved policy. Unset means the config named no `[cvm]` section to + /// derive it from, and an unfiltered node is the historical shape. + pub fn filter_policy(&self) -> &NetworkFilterConfig { + static UNFILTERED: std::sync::OnceLock<NetworkFilterConfig> = std::sync::OnceLock::new(); + self.network_filter + .as_ref() + .unwrap_or_else(|| UNFILTERED.get_or_init(NetworkFilterConfig::default)) + } + + pub fn validate(&self) -> Result<()> { + anyhow::ensure!( + self.socket_mode & !0o777 == 0, + "netd.socket_mode must contain only Unix permission bits" + ); + Ok(()) + } +} + +fn default_netd_socket() -> PathBuf { + PathBuf::from("/run/dstack/netd.sock") +} + +fn default_libvirt_uri() -> String { + "qemu:///system".to_string() +} + #[derive(Debug, Default, Clone, Deserialize, Serialize)] pub struct ProcessAnnotation { #[serde(default)] pub kind: String, #[serde(default)] pub live_for: Option<String>, + /// Whether this process's serial chardev log was opened with + /// `logappend=on`, which is what makes rotating it in place safe. + /// + /// Absent for processes launched before this option existed, and `default` + /// makes those deserialize to `false` — the conservative answer. + #[serde(default)] + pub serial_logappend: bool, } impl ProcessAnnotation { @@ -497,21 +750,275 @@ impl Config { self.run_path = self.run_path.absolutize()?.to_path_buf(); Ok(self) } + + /// Validate configuration invariants that do not require starting services + /// or modifying host state. + pub fn validate(&self) -> Result<()> { + self.host_api + .validate() + .context("Invalid host_api configuration")?; + + self.netd.validate()?; + + anyhow::ensure!(self.cvm.cid_start >= 3, "cvm.cid_start must be at least 3"); + anyhow::ensure!( + self.cvm.cid_pool_size > 0, + "cvm.cid_pool_size must be greater than zero" + ); + self.cvm + .cid_start + .checked_add(self.cvm.cid_pool_size) + .context("cvm CID pool overflows u32")?; + + anyhow::ensure!( + matches!(self.cvm.host_share_mode.as_str(), "9p" | "vhd" | "vvfat"), + "cvm.host_share_mode must be one of: 9p, vhd, vvfat" + ); + if self.cvm.auto_restart.enabled { + anyhow::ensure!( + self.cvm.auto_restart.interval > 0, + "cvm.auto_restart.interval must be greater than zero when enabled" + ); + } + for range in &self.cvm.port_mapping.range { + anyhow::ensure!( + range.from <= range.to, + "cvm.port_mapping range start {} exceeds end {}", + range.from, + range.to + ); + } + + validate_networking(&self.cvm.networking)?; + // netd creates an unfiltered TAP when the filter name is empty, which + // is what unfiltered multiqueue bridges need. Libvirt mode must never + // reach that path: it would silently produce an unbound TAP where the + // operator asked for a filtered one. + anyhow::ensure!( + self.cvm.network_filter.mode != NetworkFilterMode::Libvirt + || !self.cvm.network_filter.filter.trim().is_empty(), + "cvm.network_filter.filter must not be empty when mode is libvirt" + ); + anyhow::ensure!( + (1..=MAX_NET_QUEUES).contains(&self.cvm.max_net_queues), + "cvm.max_net_queues must be between 1 and {MAX_NET_QUEUES}" + ); + anyhow::ensure!( + !self + .cvm + .allowed_network_modes + .contains(&NetworkingMode::Custom), + "cvm.allowed_network_modes cannot contain custom" + ); + for (name, values) in [ + ("cvm.allowed_bridges", &self.cvm.allowed_bridges), + ( + "cvm.allowed_macvtap_parents", + &self.cvm.allowed_macvtap_parents, + ), + ] { + anyhow::ensure!( + values.iter().all(|value| !value.trim().is_empty()), + "{name} cannot contain empty interface names" + ); + } + anyhow::ensure!( + !self.supervisor.sock.trim().is_empty(), + "supervisor.sock must not be empty" + ); + if self.supervisor.auto_start { + for (name, value) in [ + ("supervisor.exe", self.supervisor.exe.as_str()), + ("supervisor.pid_file", self.supervisor.pid_file.as_str()), + ("supervisor.log_file", self.supervisor.log_file.as_str()), + ] { + anyhow::ensure!( + !value.trim().is_empty(), + "{name} must not be empty when supervisor.auto_start is enabled" + ); + } + } + + for (name, values) in [ + ("cvm.kms_urls", self.cvm.kms_urls.as_slice()), + ("cvm.gateway_urls", self.cvm.gateway_urls.as_slice()), + ] { + for value in values { + validate_http_url(name, value)?; + } + } + anyhow::ensure!( + self.cvm.gateway_urls.is_empty() || self.cvm.gateway_clusters.is_empty(), + "cvm.gateway_urls and cvm.gateway_clusters cannot both be configured" + ); + let mut cluster_names = std::collections::HashSet::new(); + for cluster in &self.cvm.gateway_clusters { + anyhow::ensure!( + !cluster.name.is_empty() + && cluster + .name + .bytes() + .all(|c| c.is_ascii_alphanumeric() || c == b'-' || c == b'_'), + "invalid cvm.gateway_clusters name: {}", + cluster.name + ); + anyhow::ensure!( + cluster_names.insert(cluster.name.as_str()), + "duplicate cvm.gateway_clusters name: {}", + cluster.name + ); + anyhow::ensure!( + !cluster.urls.is_empty(), + "cvm.gateway_clusters.{} must contain at least one URL", + cluster.name + ); + for url in &cluster.urls { + validate_http_url("cvm.gateway_clusters.urls", url)?; + } + } + for (name, value) in [ + ("cvm.pccs_url", Some(self.cvm.pccs_url.as_str())), + ( + "cvm.nvidia_attestation_proxy_url", + self.cvm.nvidia_attestation_proxy_url.as_deref(), + ), + ] { + if let Some(value) = value.filter(|value| !value.is_empty()) { + validate_http_url(name, value)?; + } + } + Ok(()) + } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +fn validate_http_url(name: &str, value: &str) -> Result<()> { + let url = url::Url::parse(value).with_context(|| format!("{name} contains an invalid URL"))?; + anyhow::ensure!( + matches!(url.scheme(), "http" | "https"), + "{name} URL must use http or https: {value}" + ); + anyhow::ensure!( + url.host().is_some(), + "{name} URL must include a host: {value}" + ); + Ok(()) +} + +fn validate_networking(networking: &Networking) -> Result<()> { + let prefix = networking.mac_prefix.as_str(); + if !prefix.is_empty() { + let bytes = prefix.split(':').collect::<Vec<_>>(); + anyhow::ensure!( + bytes.len() <= 3 + && bytes + .iter() + .all(|byte| byte.len() == 2 && u8::from_str_radix(byte, 16).is_ok()), + "cvm.networking.mac_prefix must contain 1 to 3 two-digit hexadecimal bytes" + ); + } + anyhow::ensure!( + networking.nic.queues.is_none(), + "cvm.networking.queues is not a node setting; queue pairs follow each VM's vCPU count, \ + bounded by cvm.max_net_queues, and a deployment overrides them per NIC" + ); + // Both describe a per-VM entry's relationship to this configuration, so + // neither means anything on the node default itself. + anyhow::ensure!( + !networking.nic.inherit_mode, + "cvm.networking.inherit_mode is per-deployment state and cannot be set on the node default" + ); + anyhow::ensure!( + networking.device.is_empty(), + "cvm.networking.device is runtime state and cannot be set in configuration" + ); + match networking.nic.mode { + NetworkingMode::Bridge => anyhow::ensure!( + !networking.nic.bridge.trim().is_empty(), + "cvm.networking.bridge must not be empty in bridge mode" + ), + NetworkingMode::Custom => anyhow::ensure!( + !networking.netdev.trim().is_empty(), + "cvm.networking.netdev must not be empty in custom mode" + ), + NetworkingMode::Macvtap => { + anyhow::ensure!( + !networking.nic.parent.trim().is_empty(), + "cvm.networking.parent must not be empty in macvtap mode" + ); + anyhow::ensure!( + matches!( + networking.macvtap_mode.as_str(), + "" | "private" | "bridge" | "vepa" | "passthru" + ), + "cvm.networking.macvtap_mode must be private, bridge, vepa, or passthru" + ); + } + // User mode has no identity fields of its own to check. + NetworkingMode::User => {} + } + Ok(()) +} + +fn default_allowed_network_modes() -> Vec<NetworkingMode> { + vec![NetworkingMode::User, NetworkingMode::Bridge] +} + +fn default_max_net_queues() -> u32 { + DEFAULT_MAX_NET_QUEUES +} + +/// Where the vCPU-scaled default stops growing. Each queue pair costs a host +/// vhost thread and two MSI-X vectors, and cross-vCPU wakeups are expensive +/// under TDX, so the benefit runs out well before a large VM's vCPU count. +/// Raising `cvm.max_net_queues` lets a deployment ask for more; it does not +/// move this, because a bigger VM should not silently get a worse default. +pub const DEFAULT_QUEUE_SCALING_CAP: u32 = 16; + +/// Default ceiling on what a deployment RPC caller may request. +pub const DEFAULT_MAX_NET_QUEUES: u32 = 16; + +/// Hard bound on queue pairs from any source, well below anything QEMU or the +/// guest driver would refuse. It exists so a malformed or hostile request +/// cannot ask the host kernel for an unbounded device, not because 64 is a +/// property of virtio-net. +pub const MAX_NET_QUEUES: u32 = 64; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "lowercase")] pub enum NetworkingMode { + /// The backend that needs nothing from the host, so it is what a NIC that + /// names none falls back to. + #[default] User, Bridge, Custom, + Macvtap, } -/// Flat networking configuration. The `mode` field selects which backend is -/// active; the remaining fields are only relevant for their respective mode -/// and carry serde defaults so they can be omitted in the config file. -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct Networking { +impl NetworkingMode { + /// The name this mode is written as in `vmm.toml`, in the RPC, and in + /// anything an operator reads. + pub fn as_str(self) -> &'static str { + match self { + NetworkingMode::User => "user", + NetworkingMode::Bridge => "bridge", + NetworkingMode::Custom => "custom", + NetworkingMode::Macvtap => "macvtap", + } + } +} + +/// What a single NIC pins: the fields a deployment may name, a VM's manifest +/// stores, and `GetInfo` reports back. +/// +/// Separate from [`Networking`] because the node's `[cvm.networking]` is not a +/// NIC -- it is a NIC *plus* the backend settings only the node may set. When +/// the two were one type, resolution copied the whole node value into every +/// VM, so a bridge NIC's manifest entry carried whatever macvtap parent the +/// node happened to have configured, and every consumer that asked "what does +/// this VM pin?" had to know which fields to ignore. Several did not. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct NicNetworking { pub mode: NetworkingMode, // ── Bridge fields ────────────────────────────────────────────── @@ -519,9 +1026,54 @@ pub struct Networking { #[serde(default)] pub bridge: String, - /// Enable userspace port forwarding for bridge-mode VMs. + // ── Macvtap fields ──────────────────────────────────────────── + /// Parent host interface for macvtap (e.g., "eth0"). #[serde(default)] - pub forward_service_enabled: bool, + pub parent: String, + + // ── Data plane tuning ────────────────────────────────────────── + /// Move packet processing from the QEMU main loop into the host kernel's + /// vhost-net data plane. `None` inherits the node default. Ignored by the + /// user-mode backend, which has no vhost support, and by custom mode, + /// which owns its whole netdev string. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vhost: Option<bool>, + /// virtio-net queue pairs. `None` scales with the VM's vCPU count. Only a + /// deployment sets this; there is no node-wide value, because the useful + /// number depends on the VM rather than the host. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queues: Option<u32>, + + // ── Ownership markers ────────────────────────────────────────── + /// Take `mode` from node configuration at every launch instead of from + /// this entry. + /// + /// A deployment that only tunes the data plane never named a backend, so + /// the node still owns which one this NIC uses. `mode` is not an `Option` + /// -- every consumer matches on it -- so the entry carries the node's + /// current mode and this flag says not to trust it across a node + /// configuration change. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub inherit_mode: bool, +} + +/// `[cvm.networking]`, and the resolved value a launch hands to QEMU: one NIC +/// plus the backend settings that belong to the node rather than to any VM. +/// +/// Resolution produces this same type because a launch needs both halves; what +/// it must never do is hand the node half back to a VM to store. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct Networking { + #[serde(flatten)] + pub nic: NicNetworking, + + // ── Macvtap fields ──────────────────────────────────────────── + /// macvtap forwarding mode. Empty selects "private". + #[serde(default)] + pub macvtap_mode: String, + /// Runtime-only character device returned by netd. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub device: String, // ── MAC prefix ───────────────────────────────────────────────── /// Fixed MAC address prefix (0-3 colon-separated hex bytes, e.g. "02:ab:cd"). @@ -540,11 +1092,74 @@ pub struct Networking { // ── Custom fields ────────────────────────────────────────────── #[serde(default)] pub netdev: String, + // ── Runtime markers ──────────────────────────────────────────── } impl Networking { pub fn is_bridge(&self) -> bool { - self.mode == NetworkingMode::Bridge + self.nic.mode == NetworkingMode::Bridge + } + + /// Whether the vhost-net data plane applies to this interface. + /// + /// Defaults to disabled: an upgraded node keeps the exact device shape its + /// VMs booted with (one queue pair, userspace virtio) until the operator + /// opts in, because `vhost = true` requires `/dev/vhost-net` to be + /// accessible to the account QEMU runs under — a precondition the VMM + /// cannot verify on the operator's behalf. + pub fn vhost_enabled(&self) -> bool { + self.nic.vhost.unwrap_or(false) && self.supports_vhost() + } + + /// Whether the backend selected by `mode` can carry a vhost-net data plane + /// at all. Custom mode is excluded because the operator supplies the whole + /// netdev string, including any vhost options. + pub fn supports_vhost(&self) -> bool { + Self::mode_supports_vhost(self.nic.mode) + } + + /// The same question about a mode on its own, for a caller deciding + /// whether a request it has not built an entry for yet can be honoured. + pub fn mode_supports_vhost(mode: NetworkingMode) -> bool { + matches!(mode, NetworkingMode::Bridge | NetworkingMode::Macvtap) + } + + /// Whether the backend selected by `mode` can carry more than one queue + /// pair. + /// + /// Custom mode is excluded for the same reason as vhost: the operator + /// supplies the whole netdev string and the VMM cannot edit it, so a + /// multiqueue device line would have nothing to pair with. The RPC refuses + /// such a request, but a node that switches its default to custom must not + /// be able to produce one behind the RPC's back. + pub fn supports_multiqueue(&self) -> bool { + Self::mode_supports_multiqueue(self.nic.mode) + } + + /// The same question about a mode on its own, for a caller deciding + /// whether a request it has not built an entry for yet can be honoured. + pub fn mode_supports_multiqueue(mode: NetworkingMode) -> bool { + matches!(mode, NetworkingMode::Bridge | NetworkingMode::Macvtap) + } + + /// Effective virtio-net queue pair count of a resolved NIC, never below + /// one. Resolution makes the vCPU-scaled default concrete, so an entry that + /// still carries none is read conservatively as single-queue. + pub fn queue_pairs(&self) -> u32 { + if !self.supports_multiqueue() { + return 1; + } + self.nic.queues.unwrap_or(1).max(1) + } + + /// Queue pairs a VM with this many vCPUs gets when it asks for none. + /// + /// The guest driver uses at most one queue pair per vCPU, so the default + /// follows the vCPU count up to a fixed cap. A node that lowers + /// `max_net_queues` below that cap means it, so the default follows it + /// down; raising it above the cap only widens what a caller may request. + pub fn default_queue_pairs(vcpu: u32, max_net_queues: u32) -> u32 { + vcpu.clamp(1, DEFAULT_QUEUE_SCALING_CAP.min(max_net_queues).max(1)) } /// Parse the mac_prefix into bytes. Returns 0-3 bytes. @@ -571,13 +1186,20 @@ impl HostApiConfig { /// The host API must only listen on vsock for security reasons. /// TCP/Unix socket listening is not supported. pub fn validate(&self) -> Result<()> { - if !self.address.starts_with("vsock:") { - anyhow::bail!( + let cid = self.address.strip_prefix("vsock:").with_context(|| { + format!( "Host API address must be a vsock address (e.g., 'vsock:2'), got: '{}'. \ - TCP/Unix socket listening is not supported for the host API.", + TCP/Unix socket listening is not supported for the host API.", self.address - ); + ) + })?; + if let Some(cid) = cid.strip_prefix("0x") { + u32::from_str_radix(cid, 16).context("Host API address contains an invalid CID")?; + } else { + cid.parse::<u32>() + .context("Host API address contains an invalid CID")?; } + anyhow::ensure!(self.port > 0, "Host API port must be greater than zero"); Ok(()) } } @@ -589,10 +1211,6 @@ pub struct KeyProviderConfig { pub port: u16, } -fn default_serial_history_max_bytes() -> u64 { - 4 * 1024 * 1024 // 4MB -} - const CLIENT_CONF_PATH: &str = "/etc/dstack/client.conf"; fn read_qemu_path_from_client_conf() -> Option<PathBuf> { #[derive(Debug, Deserialize)] @@ -656,21 +1274,6 @@ impl Config { } } info!("QEMU path: {}", me.cvm.qemu_path.display()); - - // Detect QEMU version if not already set - match &me.cvm.qemu_version { - None => match detect_qemu_version(&me.cvm.qemu_path) { - Ok(version) => { - info!("Detected QEMU version: {version}"); - me.cvm.qemu_version = Some(version); - } - Err(e) => { - warn!("Failed to detect QEMU version: {e}"); - // Continue without version - the system will use defaults - } - }, - Some(version) => info!("Configured QEMU version: {version}"), - } } Ok(me) } @@ -680,6 +1283,102 @@ impl Config { mod tests { use super::*; + #[cfg(unix)] + fn cvm_with_qemu(dir: &tempfile::TempDir, script: &str) -> CvmConfig { + use std::os::unix::fs::PermissionsExt; + let path = dir.path().join("qemu-system-x86_64"); + std::fs::write(&path, format!("#!/bin/sh\n{script}\n")).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + let config: Config = Figment::from(load_config_figment(None)).extract().unwrap(); + CvmConfig { + qemu_path: path, + qemu_version: None, + ..config.cvm + } + } + + // Regression: a VMM staying up across a QEMU upgrade used to declare the + // version it detected at config load while executing the new binary. + #[cfg(unix)] + #[test] + fn qemu_version_follows_the_binary_at_qemu_path() { + let dir = tempfile::tempdir().unwrap(); + let cvm = cvm_with_qemu(&dir, "echo 'QEMU emulator version 10.1.0'"); + assert_eq!(cvm.resolve_qemu_version().unwrap(), "10.1.0"); + + cvm_with_qemu(&dir, "echo 'QEMU emulator version 9.2.1'"); + assert_eq!(cvm.resolve_qemu_version().unwrap(), "9.2.1"); + } + + #[cfg(unix)] + #[test] + fn explicit_qemu_version_wins_over_the_binary() { + let dir = tempfile::tempdir().unwrap(); + let mut cvm = cvm_with_qemu(&dir, "echo 'QEMU emulator version 9.2.1'"); + cvm.qemu_version = Some("8.2.2".to_string()); + assert_eq!(cvm.resolve_qemu_version().unwrap(), "8.2.2"); + } + + #[cfg(unix)] + #[test] + fn a_wrapper_banner_does_not_hide_the_version() { + let dir = tempfile::tempdir().unwrap(); + let cvm = cvm_with_qemu( + &dir, + "echo 'wrapper: exec /usr/libexec/qemu-kvm' >&2; \ + echo 'QEMU emulator version 9.2.1'", + ); + assert_eq!(cvm.resolve_qemu_version().unwrap(), "9.2.1"); + } + + // A banner carrying a version-shaped word must not be mistaken for the + // QEMU version: a wrong guess boots a CVM that cannot be attested. + #[cfg(unix)] + #[test] + fn an_undetectable_qemu_version_fails_the_start() { + let dir = tempfile::tempdir().unwrap(); + let cvm = cvm_with_qemu(&dir, "echo 'wrapper: exec /usr/libexec/qemu-kvm-8.2.0'"); + let err = format!("{:#}", cvm.resolve_qemu_version().unwrap_err()); + assert!(err.contains("set `qemu_version` in vmm.toml"), "{err}"); + } + + #[test] + fn auto_restart_config_rejects_hot_loop_and_inverted_backoff() { + let mut config = AutoRestartConfig { + enabled: true, + interval: 0, + max_retries: 3, + initial_backoff: 2, + max_backoff: 5, + reset_window: 10, + }; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("interval")); + config.interval = 1; + config.initial_backoff = 0; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("initial_backoff")); + config.initial_backoff = 6; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("max_backoff")); + config.max_backoff = 6; + assert!(config.validate().is_ok()); + + config.enabled = false; + config.interval = 0; + config.initial_backoff = 7; + assert!(config.validate().is_ok()); + } + #[test] fn test_parse_qemu_version_debian_format() { let output = "QEMU emulator version 8.2.2 (Debian 2:8.2.2+ds-0ubuntu1.4+tdx1.0)\nCopyright (c) 2003-2023 Fabrice Bellard and the QEMU Project developers"; @@ -709,10 +1408,9 @@ mod tests { } #[test] - fn test_parse_qemu_version_fallback() { + fn test_parse_qemu_version_without_qemu_wording() { let output = "Some unusual format 8.1.5 with version info"; - let version = parse_qemu_version_from_output(output).unwrap(); - assert_eq!(version, "8.1.5"); + assert!(parse_qemu_version_from_output(output).is_err()); } #[test] @@ -724,8 +1422,8 @@ mod tests { #[test] fn tee_platform_deserializes_amd_sev_snp() { - let platform: TeePlatform = serde_json::from_str("\"amd-sev-snp\"").unwrap(); - assert_eq!(platform, TeePlatform::AmdSevSnp); + let platform: CvmPlatform = serde_json::from_str("\"amd-sev-snp\"").unwrap(); + assert_eq!(platform, CvmPlatform::AmdSevSnp); } #[test] @@ -733,17 +1431,17 @@ mod tests { #[derive(Deserialize)] struct Wrap { #[serde(default, deserialize_with = "deserialize_platform")] - platform: Option<TeePlatform>, + platform: Option<CvmPlatform>, } let parse = |s: &str| serde_json::from_str::<Wrap>(s).unwrap().platform; // Omitted and the legacy `auto` literal both mean "auto-detect" (None). assert_eq!(parse("{}"), None); assert_eq!(parse(r#"{"platform":"auto"}"#), None); // Explicit platforms are pinned. - assert_eq!(parse(r#"{"platform":"tdx"}"#), Some(TeePlatform::Tdx)); + assert_eq!(parse(r#"{"platform":"tdx"}"#), Some(CvmPlatform::Tdx)); assert_eq!( parse(r#"{"platform":"amd-sev-snp"}"#), - Some(TeePlatform::AmdSevSnp) + Some(CvmPlatform::AmdSevSnp) ); } @@ -756,50 +1454,272 @@ mod tests { use dstack_types::TdxAttestationVariant::{Legacy, Lite}; - // Explicit settings bypass auto heuristics. - assert_eq!( - TdxAttestationVariantConfig::Legacy.resolve(2048, true), - Legacy - ); - assert_eq!(TdxAttestationVariantConfig::Lite.resolve(1024, false), Lite); + // Explicit settings bypass auto entirely. + assert_eq!(TdxAttestationVariantConfig::Legacy.resolve(true), Legacy); + assert_eq!(TdxAttestationVariantConfig::Lite.resolve(false), Lite); - // Auto avoids lite for sub-3 GiB memory sizes except exactly 2 GiB. - assert_eq!( - TdxAttestationVariantConfig::Auto.resolve(1024, true), - Legacy - ); - assert_eq!( - TdxAttestationVariantConfig::Auto.resolve(2816, true), - Legacy - ); - assert_eq!(TdxAttestationVariantConfig::Auto.resolve(2048, true), Lite); + // Auto follows image support alone; memory no longer takes part. + assert_eq!(TdxAttestationVariantConfig::Auto.resolve(true), Lite); + assert_eq!(TdxAttestationVariantConfig::Auto.resolve(false), Legacy); + } + + fn default_config() -> Config { + use rocket::figment::providers::{Format, Toml}; - // At 3 GiB and above, auto follows image support. - assert_eq!(TdxAttestationVariantConfig::Auto.resolve(3072, true), Lite); + Figment::from(Toml::string(DEFAULT_CONFIG)) + .extract() + .expect("default VMM config should parse") + } + + /// The ownership marker is additive on disk: manifests and runtime network + /// snapshots written before it existed still load, and an entry that does + /// not carry it serializes exactly as it used to. + #[test] + fn ownership_markers_are_omitted_when_unset_and_default_when_absent() { + let mut networking: Networking = + serde_json::from_str(r#"{"mode":"bridge","bridge":"br0"}"#).unwrap(); + assert!(!networking.nic.inherit_mode); + + let json = serde_json::to_string(&networking).unwrap(); + assert!(!json.contains("inherit_mode"), "{json}"); + + networking.nic.inherit_mode = true; + let json = serde_json::to_string(&networking).unwrap(); + assert!(json.contains(r#""inherit_mode":true"#), "{json}"); assert_eq!( - TdxAttestationVariantConfig::Auto.resolve(3072, false), - Legacy + serde_json::from_str::<Networking>(&json).unwrap(), + networking ); } + #[test] + fn config_validation_accepts_defaults() { + let config = default_config(); + assert_eq!(config.netd.socket_mode, 0o660); + assert!(config.cvm.shuffle_kms_urls); + assert!(config.cvm.shuffle_gateway_urls); + config.validate().unwrap(); + } + + #[test] + fn config_validation_rejects_invalid_static_invariants() { + let mut config = default_config(); + config.netd.socket_mode = 0o1660; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("socket_mode")); + + let mut config = default_config(); + config.cvm.cid_pool_size = 0; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("cid_pool_size")); + + let mut config = default_config(); + config.cvm.port_mapping.range[0].from = 200; + config.cvm.port_mapping.range[0].to = 100; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("range start")); + + // An empty filter tells netd to create an unfiltered TAP, so libvirt + // mode must never carry one. + let mut config = default_config(); + config.cvm.network_filter.mode = NetworkFilterMode::Libvirt; + config.cvm.network_filter.filter = String::new(); + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("network_filter.filter")); + + let mut config = default_config(); + config.cvm.max_net_queues = 0; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("max_net_queues")); + + let mut config = default_config(); + config.cvm.max_net_queues = MAX_NET_QUEUES + 1; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("max_net_queues")); + + // The node-wide value is gone; say so rather than ignoring it. + let mut config = default_config(); + config.cvm.networking.nic.queues = Some(4); + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("cvm.networking.queues is not a node setting")); + + let mut config = default_config(); + config.cvm.networking.mac_prefix = "02:not-hex".into(); + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("mac_prefix")); + + let mut config = default_config(); + config.cvm.host_share_mode = "unknown".into(); + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("host_share_mode")); + } + + #[test] + fn config_validation_rejects_invalid_endpoints() { + let mut config = default_config(); + config.cvm.kms_urls = vec!["not a URL".into()]; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("kms_urls")); + + let mut config = default_config(); + config.supervisor.sock.clear(); + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("supervisor.sock")); + + let mut config = default_config(); + config.cvm.networking.nic.mode = NetworkingMode::Bridge; + config.cvm.networking.nic.bridge.clear(); + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("networking.bridge")); + + let mut config = default_config(); + config.host_api.address = "vsock:not-a-cid".into(); + assert!(format!("{:#}", config.validate().unwrap_err()).contains("invalid CID")); + } + + #[test] + fn config_validation_rejects_mixed_gateway_syntax() { + let mut config = default_config(); + config.cvm.gateway_urls = vec!["https://legacy-gateway.example.com".into()]; + config.cvm.gateway_clusters = vec![dstack_types::GatewayClusterConfig { + name: "primary".into(), + urls: vec!["https://gateway.example.com".into()], + }]; + let error = config.validate().unwrap_err(); + assert!(error + .to_string() + .contains("gateway_urls and cvm.gateway_clusters cannot both")); + } + + #[test] + fn config_validation_does_not_require_supervisor_startup_paths_when_disabled() { + let mut config = default_config(); + config.supervisor.auto_start = false; + config.supervisor.exe.clear(); + config.supervisor.pid_file.clear(); + config.supervisor.log_file.clear(); + config.validate().unwrap(); + } + #[test] fn tee_platform_auto_detects_amd_sev_snp_from_flag() { let cpuinfo = "flags : fpu svm sev sev_es sev_snp debug_swap"; assert_eq!( - TeePlatform::resolve_from_cpuinfo(cpuinfo), - TeePlatform::AmdSevSnp + CvmPlatform::resolve_from_cpuinfo(cpuinfo), + CvmPlatform::AmdSevSnp ); } #[test] fn tee_platform_auto_detects_tdx_host() { let cpuinfo = "flags : fpu vmx tdx_host_platform"; - assert_eq!(TeePlatform::resolve_from_cpuinfo(cpuinfo), TeePlatform::Tdx); + assert_eq!(CvmPlatform::resolve_from_cpuinfo(cpuinfo), CvmPlatform::Tdx); } #[test] fn tee_platform_auto_falls_back_to_tdx_without_tee_flag() { let cpuinfo = "flags : fpu vmx"; - assert_eq!(TeePlatform::resolve_from_cpuinfo(cpuinfo), TeePlatform::Tdx); + assert_eq!(CvmPlatform::resolve_from_cpuinfo(cpuinfo), CvmPlatform::Tdx); + } +} + +#[cfg(test)] +mod networking_shape_tests { + use super::{Config, Networking, NetworkingMode, NicNetworking, DEFAULT_CONFIG}; + + /// Splitting the type must not split the wire format. `[cvm.networking]` + /// is flattened, so a node config, a stored manifest and a runtime-networks + /// snapshot written by an earlier build all still parse, and what this + /// build writes is byte-for-byte what the old one did. + #[test] + fn the_split_types_keep_one_flat_serialized_shape() { + let legacy = serde_json::json!({ + "mode": "bridge", + "bridge": "br0", + "parent": "eth0", + "macvtap_mode": "private", + "device": "/dev/tap7", + "mac_prefix": "02:aa:bb", + "net": "10.0.2.0/24", + "dhcp_start": "10.0.2.15", + "restrict": true, + "netdev": "", + "vhost": false, + "queues": 4, + "inherit_mode": true, + }); + + // A resolved value keeps every field, at the same names as before. + let resolved: Networking = serde_json::from_value(legacy.clone()).unwrap(); + assert_eq!(resolved.nic.mode, NetworkingMode::Bridge); + assert_eq!(resolved.nic.bridge, "br0"); + assert_eq!(resolved.nic.queues, Some(4)); + assert!(resolved.nic.inherit_mode); + assert_eq!(resolved.macvtap_mode, "private"); + assert_eq!(resolved.net, "10.0.2.0/24"); + assert!(resolved.restrict); + let round_tripped = serde_json::to_value(&resolved).unwrap(); + assert_eq!(round_tripped["mode"], "bridge"); + assert_eq!(round_tripped["macvtap_mode"], "private"); + assert_eq!(round_tripped["queues"], 4); + + // A manifest entry written by a build that stored the whole thing + // still loads; the node's half is simply dropped on the way in. + let pinned: NicNetworking = serde_json::from_value(legacy).unwrap(); + assert_eq!(pinned.bridge, "br0"); + assert_eq!(pinned.parent, "eth0"); + assert_eq!(pinned.vhost, Some(false)); + let stored = serde_json::to_value(&pinned).unwrap(); + assert_eq!(stored.as_object().unwrap().len(), 6); + assert!(stored.get("macvtap_mode").is_none()); + assert!(stored.get("net").is_none()); + } + + /// The TOML section still deserializes through the flatten. + #[test] + fn the_node_section_still_parses_from_toml() { + use rocket::figment::providers::Format as _; + let config: Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + assert_eq!(config.cvm.networking.nic.mode, NetworkingMode::User); } } diff --git a/dstack/vmm/src/console_v0.html b/dstack/vmm/src/console_v0.html deleted file mode 100644 index d68227724..000000000 --- a/dstack/vmm/src/console_v0.html +++ /dev/null @@ -1,2631 +0,0 @@ -<!-- -SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network> - -SPDX-License-Identifier: Apache-2.0 ---> - -<!DOCTYPE html> -<html lang="en"> - -<head> - <meta charset="UTF-8"> - <meta name="viewport" content="width=device-width, initial-scale=1.0"> - <title>{{TITLE}} - - - - - - - - - - - - - - - - - - - - -
- - -
-
-

Deploy a new instance

-
-
-
- - -
- -
- - -
- -
- - -
- -
- -
- - -
-
- -
- -
- - -
- Leave as 0 to disable swap. -
- -
- - -
- -
- - -
- -
- -
-
- - or paste below - -
- -
-
-
- -
- - - -
- -
- - -
- -
- - -
- -
- - -
- -
- -
- -
- -
- - - - - - - - -
-
-
- - -
- -
- -
- -
- -
-
- - -
-
- - -
-
-
-
- Compose Hash: 0x{{ composeHashPreview }} -
- -
- - -
-
-
-
- -

VM List

-
-
-
- - -
-
- Total: {{ totalVMs }} -
-
- - - - - - - - - - - - - -
NameStatusUptimeViewActions
-
- - -
- Page - - of {{ maxPage || 1 }} -
- - - - - -
-
- -
-
-

Update VM Config

- -
-
- - -
-
- -
- - -
-
-
- -
- -
- - -
- Enable "Update compose" to change swap size. -
- -
- - -
- -
-
- - -
- - -
-
- -
- -
- - -
-
-
- -
-
-
- -
-
- - or paste below - -
- -
-
-
- - -
-
- Compose Hash: 0x{{ upgradeComposeHashPreview }} -
-
-
- -
-
- -
-
- -
- -
- -
- - -
- -
- - -
-
-
- -
-
-

Derive VM

-

- This will create a new VM instance with the same app id, but the disk state will NOT migrate to the - new instance. -

- -
- - -
- -
- - -
-
- - -
-
- - -
-
- - -
-
-
- -
-
- -
-
-
- {{ errorMessage }} - -
-
-
- Version: v{{ version.version }} ({{ version.rev }}) -
-
- - - - - - - - - - - - - diff --git a/dstack/vmm/src/discovery.rs b/dstack/vmm/src/discovery.rs index 8a2440f39..181e7da52 100644 --- a/dstack/vmm/src/discovery.rs +++ b/dstack/vmm/src/discovery.rs @@ -8,6 +8,7 @@ //! directory so that CLI tools can discover all running instances on the host. use anyhow::{Context, Result}; +use fs_err as fs; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use tracing::{info, warn}; @@ -99,7 +100,7 @@ impl DiscoveryRegistration { impl Drop for DiscoveryRegistration { fn drop(&mut self) { - match std::fs::remove_file(&self.path) { + match fs::remove_file(&self.path) { Ok(()) => info!("unregistered VMM instance at {}", self.path.display()), Err(e) => warn!( "failed to remove discovery file {}: {e}", @@ -112,7 +113,7 @@ impl Drop for DiscoveryRegistration { /// Clean up stale discovery files from dead processes. pub fn cleanup_stale_registrations() { let dir = discovery_dir(); - let entries = match std::fs::read_dir(dir) { + let entries = match fs::read_dir(dir) { Ok(e) => e, Err(_) => return, }; @@ -122,7 +123,7 @@ pub fn cleanup_stale_registrations() { if path.extension().and_then(|e| e.to_str()) != Some("json") { continue; } - let content = match std::fs::read_to_string(&path) { + let content = match fs::read_to_string(&path) { Ok(c) => c, Err(_) => continue, }; @@ -138,7 +139,7 @@ pub fn cleanup_stale_registrations() { info.pid, path.display() ); - let _ = std::fs::remove_file(&path); + let _ = fs::remove_file(&path); } } } diff --git a/dstack/vmm/src/gpu_reset.rs b/dstack/vmm/src/gpu_reset.rs new file mode 100644 index 000000000..e58876b9a --- /dev/null +++ b/dstack/vmm/src/gpu_reset.rs @@ -0,0 +1,299 @@ +// SPDX-FileCopyrightText: © 2024-2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + collections::BTreeSet, + fs::{File, OpenOptions}, + os::unix::fs::FileExt, + path::{Path, PathBuf}, + thread, + time::{Duration, Instant}, +}; + +use anyhow::{bail, Context, Result}; +use tracing::info; + +use crate::{app::GpuConfig, config::GpuConfig as HostGpuConfig}; + +const PCI_SYSFS_DEVICES: &str = "/sys/bus/pci/devices"; +const PCI_BRIDGE_CLASS: u32 = 0x0604; +const PCI_BRIDGE_CONTROL: u64 = 0x3e; +const PCI_BRIDGE_CTL_BUS_RESET: u16 = 1 << 6; +const SBR_ASSERT_TIME: Duration = Duration::from_millis(100); +const SBR_POLL_INTERVAL: Duration = Duration::from_millis(100); +const SBR_STABLE_TIME: Duration = Duration::from_millis(500); + +/// Clears device-internal state that can survive VFIO's attach-time FLR. +/// +/// Each selected GPU must sit alone behind a dedicated PCIe bridge. Resetting +/// a shared bridge could disrupt unrelated devices, so that topology is +/// rejected rather than guessed at. +pub fn sanitize_on_attach(host: &HostGpuConfig, devices: &GpuConfig) -> Result<()> { + if !host.enabled || !host.sanitize_on_attach || devices.gpus.is_empty() { + return Ok(()); + } + sanitize_at( + Path::new(PCI_SYSFS_DEVICES), + devices, + Duration::from_millis(host.sbr_timeout_ms), + ) +} + +fn sanitize_at(sysfs_devices: &Path, devices: &GpuConfig, timeout: Duration) -> Result<()> { + let selected = devices + .gpus + .iter() + .map(|gpu| normalize_slot(&gpu.slot)) + .collect::>(); + let mut bridges = BTreeSet::new(); + + for gpu in &selected { + let bridge = upstream_bridge(sysfs_devices, gpu)?; + ensure_dedicated_bridge(&bridge, gpu)?; + bridges.insert(bridge); + } + + for bridge in bridges { + info!(bridge = %bridge.display(), "sanitizing GPU with PCIe Secondary Bus Reset"); + secondary_bus_reset(&bridge) + .with_context(|| format!("failed to sanitize GPU using bridge {}", bridge.display()))?; + } + wait_for_vfio_ready(sysfs_devices, &selected, timeout)?; + Ok(()) +} + +fn wait_for_vfio_ready( + sysfs_devices: &Path, + devices: &BTreeSet, + timeout: Duration, +) -> Result<()> { + let drivers_probe = sysfs_devices + .parent() + .context("PCI sysfs devices directory has no parent")? + .join("drivers_probe"); + let deadline = Instant::now() + timeout; + let mut stable_since = None; + loop { + let not_ready = devices + .iter() + .filter(|slot| { + let device = sysfs_devices.join(slot); + if !driver_is_vfio(&device) { + let _ = fs_err::write(&drivers_probe, format!("{slot}\n")); + } + !is_vfio_ready(&device) + }) + .cloned() + .collect::>(); + if not_ready.is_empty() { + let since = *stable_since.get_or_insert_with(Instant::now); + if since.elapsed() >= SBR_STABLE_TIME { + info!(count = devices.len(), "all sanitized GPUs are VFIO-ready"); + return Ok(()); + } + } else { + stable_since = None; + } + if Instant::now() >= deadline { + bail!( + "timed out after {} ms waiting for sanitized GPUs to become VFIO-ready: {}", + timeout.as_millis(), + not_ready.join(", ") + ); + } + thread::sleep(SBR_POLL_INTERVAL); + } +} + +fn is_vfio_ready(device: &Path) -> bool { + let Ok(device) = device.canonicalize() else { + return false; + }; + let Ok(config) = File::open(device.join("config")) else { + return false; + }; + let Ok(vendor) = read_u16(&config, 0) else { + return false; + }; + if matches!(vendor, 0 | 0xffff) { + return false; + } + driver_is_vfio(&device) + && device.join("iommu_group").canonicalize().is_ok() + && fs_err::read_dir(device.join("vfio-dev")) + .ok() + .and_then(|mut entries| entries.next()) + .is_some() +} + +fn driver_is_vfio(device: &Path) -> bool { + device + .join("driver") + .canonicalize() + .ok() + .and_then(|path| path.file_name().map(|name| name == "vfio-pci")) + .unwrap_or(false) +} + +fn normalize_slot(slot: &str) -> String { + if slot.matches(':').count() == 1 { + format!("0000:{slot}") + } else { + slot.to_owned() + } +} + +fn upstream_bridge(sysfs_devices: &Path, slot: &str) -> Result { + let device_link = sysfs_devices.join(slot); + let device = device_link + .canonicalize() + .with_context(|| format!("failed to resolve PCI device {slot}"))?; + let bridge = device + .parent() + .context("PCI device has no upstream bridge")? + .to_path_buf(); + let class = fs_err::read_to_string(bridge.join("class")) + .with_context(|| format!("failed to read class of bridge {}", bridge.display()))?; + let class = u32::from_str_radix(class.trim().trim_start_matches("0x"), 16) + .context("invalid PCI bridge class")?; + if class >> 8 != PCI_BRIDGE_CLASS { + bail!("GPU {slot} is not directly behind a PCI bridge (parent class {class:#08x})"); + } + Ok(bridge) +} + +fn ensure_dedicated_bridge(bridge: &Path, gpu: &str) -> Result<()> { + let mut downstream = Vec::new(); + for entry in fs_err::read_dir(bridge)? { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name != gpu && is_pci_slot(&name) && entry.file_type()?.is_dir() { + downstream.push(name.into_owned()); + } + } + if !downstream.is_empty() { + bail!( + "refusing to reset shared bridge {} for GPU {gpu}; other downstream PCI devices: {}", + bridge.display(), + downstream.join(", ") + ); + } + Ok(()) +} + +fn is_pci_slot(name: &str) -> bool { + name.len() == 12 + && name.as_bytes()[4] == b':' + && name.as_bytes()[7] == b':' + && name.as_bytes()[10] == b'.' + && name + .chars() + .enumerate() + .all(|(i, c)| matches!(i, 4 | 7 | 10) || c.is_ascii_hexdigit()) +} + +fn secondary_bus_reset(bridge: &Path) -> Result<()> { + let config_path = bridge.join("config"); + let config = OpenOptions::new() + .read(true) + .write(true) + .open(&config_path) + .with_context(|| format!("failed to open {}", config_path.display()))?; + let original = read_u16(&config, PCI_BRIDGE_CONTROL)?; + write_u16( + &config, + PCI_BRIDGE_CONTROL, + original | PCI_BRIDGE_CTL_BUS_RESET, + )?; + thread::sleep(SBR_ASSERT_TIME); + write_u16( + &config, + PCI_BRIDGE_CONTROL, + original & !PCI_BRIDGE_CTL_BUS_RESET, + )?; + Ok(()) +} + +fn read_u16(file: &File, offset: u64) -> Result { + let mut value = [0_u8; 2]; + file.read_exact_at(&mut value, offset)?; + Ok(u16::from_le_bytes(value)) +} + +fn write_u16(file: &File, offset: u64, value: u16) -> Result<()> { + file.write_all_at(&value.to_le_bytes(), offset)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::symlink; + + use crate::app::{AttachMode, GpuSpec}; + + #[test] + fn normalizes_short_pci_slots() { + assert_eq!(normalize_slot("0f:00.0"), "0000:0f:00.0"); + assert_eq!(normalize_slot("0001:0f:00.0"), "0001:0f:00.0"); + } + + #[test] + fn recognizes_pci_slots() { + assert!(is_pci_slot("0000:0f:00.0")); + assert!(!is_pci_slot("class")); + assert!(!is_pci_slot("0000:0g:00.0")); + } + + #[test] + fn skips_sanitization_when_gpu_passthrough_is_disabled() { + let host = HostGpuConfig { + enabled: false, + listing: Vec::new(), + exclude: Vec::new(), + include: Vec::new(), + allow_attach_all: true, + sanitize_on_attach: true, + sbr_timeout_ms: 10_000, + }; + let devices = GpuConfig { + attach_mode: AttachMode::Listed, + gpus: vec![GpuSpec { + slot: "ff:00.0".into(), + }], + bridges: Vec::new(), + }; + + sanitize_on_attach(&host, &devices).unwrap(); + } + + #[test] + fn finds_a_dedicated_upstream_bridge() { + let root = tempfile::tempdir().unwrap(); + let bridge = root.path().join("tree/0000:00:01.0"); + let gpu = bridge.join("0000:01:00.0"); + fs_err::create_dir_all(&gpu).unwrap(); + fs_err::write(bridge.join("class"), "0x060400\n").unwrap(); + let devices = root.path().join("devices"); + fs_err::create_dir(&devices).unwrap(); + symlink(&gpu, devices.join("0000:01:00.0")).unwrap(); + + let found = upstream_bridge(&devices, "0000:01:00.0").unwrap(); + assert_eq!(found, bridge.canonicalize().unwrap()); + ensure_dedicated_bridge(&found, "0000:01:00.0").unwrap(); + } + + #[test] + fn rejects_a_bridge_shared_with_another_device() { + let root = tempfile::tempdir().unwrap(); + let bridge = root.path().join("0000:00:01.0"); + fs_err::create_dir_all(bridge.join("0000:01:00.0")).unwrap(); + fs_err::create_dir_all(bridge.join("0000:01:00.1")).unwrap(); + + let error = ensure_dedicated_bridge(&bridge, "0000:01:00.0").unwrap_err(); + assert!(error.to_string().contains("shared bridge")); + assert!(error.to_string().contains("0000:01:00.1")); + } +} diff --git a/dstack/vmm/src/guest_api_service.rs b/dstack/vmm/src/guest_api_service.rs index 1fc089f8f..439df00d1 100644 --- a/dstack/vmm/src/guest_api_service.rs +++ b/dstack/vmm/src/guest_api_service.rs @@ -6,7 +6,7 @@ use crate::App as AppState; use anyhow::Result; use guest_api::{ proxied_guest_api_server::{ProxiedGuestApiRpc, ProxiedGuestApiServer}, - GuestInfo, Id, ListContainersResponse, NetworkInformation, SystemInfo, + GpuInfoResponse, GuestInfo, Id, ListContainersResponse, NetworkInformation, SystemInfo, }; use ra_rpc::{CallContext, RpcCall}; use std::ops::Deref; @@ -42,6 +42,10 @@ impl ProxiedGuestApiRpc for GuestApiHandler { self.guest_agent_client(&request.id)?.sys_info().await } + async fn gpu_info(self, request: Id) -> Result { + self.guest_agent_client(&request.id)?.gpu_info().await + } + async fn network_info(self, request: Id) -> Result { self.guest_agent_client(&request.id)?.network_info().await } diff --git a/dstack/vmm/src/logrotate.rs b/dstack/vmm/src/logrotate.rs new file mode 100644 index 000000000..fb004109c --- /dev/null +++ b/dstack/vmm/src/logrotate.rs @@ -0,0 +1,307 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Logrotate-style rotation for files a live process holds open. +//! +//! Like the logrotate(8) this is modelled on, rotation works on a path and does +//! not care what the log contains or who writes it: `.N-1` becomes +//! `.N`, `` is archived as `.1`, and the oldest segment is +//! discarded. +//! +//! # The one requirement on callers +//! +//! The writer must hold the file open with `O_APPEND`. +//! +//! Rotation truncates the live file in place rather than renaming it. Renaming +//! would leave the writer appending into an unlinked inode, so its output would +//! vanish silently. Truncating keeps the writer's fd valid — but only under +//! `O_APPEND`, which forces every write to the current end of file. A writer +//! without it keeps writing at its stale offset, so the truncation punches a +//! sparse hole and the file springs straight back to its previous size. The cap +//! then never holds and every later check rotates again. +//! +//! For QEMU chardev logs that means `logappend=on`; for a process's stdout or +//! stderr, an `OpenOptions::append(true)` sink. + +use std::path::{Path, PathBuf}; + +use fs_err as fs; +use tracing::warn; + +/// Outcome of a rotation. +pub struct Rotated { + /// Bytes removed from the live log. + pub bytes: u64, + /// Where those bytes were kept, or `None` if they were discarded because + /// no backups are retained. + pub archived: Option, +} + +/// Path of rotated segment `index`, `1` being the most recent. +pub fn segment_path(path: &Path, index: usize) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(format!(".{index}")); + PathBuf::from(name) +} + +/// Empty `path` in place, keeping the writer's open fd valid. +/// +/// Missing files are not an error: callers use this to guarantee a log starts +/// empty without having to care whether it exists yet. +pub fn truncate(path: &Path) { + if !path.exists() { + return; + } + if let Err(err) = fs::write(path, b"") { + warn!("failed to truncate {}: {err}", path.display()); + } +} + +/// Rotate `path`, discarding the oldest of `max_backups` segments. +/// +/// Returns `None` when there was nothing to rotate, which includes an empty or +/// missing log — rotating those would spend a segment slot on an empty file and +/// push a segment that still has content off the end. +pub fn rotate(path: &Path, max_backups: usize) -> Option { + let bytes = match fs::metadata(path) { + Ok(meta) if meta.len() > 0 => meta.len(), + _ => return None, + }; + if max_backups == 0 { + truncate(path); + return Some(Rotated { + bytes, + archived: None, + }); + } + + let oldest = segment_path(path, max_backups); + if oldest.exists() { + if let Err(err) = fs::remove_file(&oldest) { + warn!( + "failed to remove oldest segment {}: {err}", + oldest.display() + ); + return None; + } + } + for index in (1..max_backups).rev() { + let from = segment_path(path, index); + if !from.exists() { + continue; + } + if let Err(err) = fs::rename(&from, segment_path(path, index + 1)) { + warn!("failed to shift segment {}: {err}", from.display()); + return None; + } + } + + // Copy rather than rename: the writer holds an open fd on the live log. + let archived = segment_path(path, 1); + if let Err(err) = fs::copy(path, &archived) { + warn!("failed to archive {}: {err}", path.display()); + return None; + } + truncate(path); + Some(Rotated { + bytes, + archived: Some(archived), + }) +} + +/// Rotate `path` if it has grown past `max_bytes`. `max_bytes == 0` disables +/// rotation entirely. +pub fn rotate_if_oversized(path: &Path, max_bytes: u64, max_backups: usize) -> Option { + if max_bytes == 0 { + return None; + } + match fs::metadata(path) { + Ok(meta) if meta.len() > max_bytes => {} + _ => return None, + } + rotate(path, max_backups) +} + +/// Record in the freshly emptied log where its previous content went. +/// +/// A reader that only ever sees the live file — as the VMM log API does — would +/// otherwise find it empty with no explanation. This does not make the segments +/// readable; it makes their absence self-explanatory. +pub fn append_rotation_note(path: &Path, rotated: &Rotated) { + use std::io::Write; + + let Ok(mut file) = fs::OpenOptions::new().append(true).open(path) else { + return; + }; + let timestamp = humantime::format_rfc3339_seconds(std::time::SystemTime::now()); + let bytes = rotated.bytes; + let _ = match &rotated.archived { + Some(archived) => { + let name = archived + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_else(|| archived.display().to_string()); + writeln!( + file, + "\n===== rotated {bytes} bytes to {name} @ {timestamp} =====\n" + ) + } + None => writeln!( + file, + "\n===== discarded {bytes} bytes @ {timestamp} =====\n" + ), + }; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn segment_path_appends_the_index_to_the_whole_name() { + // Suffixing the full name rather than replacing an extension keeps + // `serial.log.1` readable and works for files with no extension. + assert_eq!( + segment_path(Path::new("/run/serial.log"), 1).to_str(), + Some("/run/serial.log.1") + ); + assert_eq!( + segment_path(Path::new("/run/stdout"), 12).to_str(), + Some("/run/stdout.12") + ); + } + + #[test] + fn rotate_shifts_and_drops_the_oldest() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let log = temp.path().join("app.log"); + + for marker in ["first", "second", "third", "fourth"] { + fs::write(&log, format!("{marker}\n"))?; + assert!(rotate(&log, 3).is_some()); + } + + assert_eq!(fs::read(segment_path(&log, 1))?, b"fourth\n"); + assert_eq!(fs::read(segment_path(&log, 2))?, b"third\n"); + assert_eq!(fs::read(segment_path(&log, 3))?, b"second\n"); + assert!(!segment_path(&log, 4).exists()); + Ok(()) + } + + #[test] + fn rotate_keeps_the_live_file_inode() -> anyhow::Result<()> { + use std::os::unix::fs::MetadataExt; + let temp = tempfile::tempdir()?; + let log = temp.path().join("app.log"); + + fs::write(&log, b"before\n")?; + let before = fs::metadata(&log)?.ino(); + rotate(&log, 3); + + // Renaming the live file would leave the writer appending into an + // unlinked inode, losing every later line without an error. + assert_eq!(fs::metadata(&log)?.ino(), before); + assert_eq!(fs::read(&log)?.len(), 0); + Ok(()) + } + + #[test] + fn rotate_skips_an_empty_or_missing_log() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let log = temp.path().join("app.log"); + + // Missing: must not create anything. + assert!(rotate(&log, 3).is_none()); + assert!(!segment_path(&log, 1).exists()); + + // Empty: spending a slot here would push a segment that still has + // content off the end. + fs::write(&log, b"")?; + assert!(rotate(&log, 3).is_none()); + assert!(!segment_path(&log, 1).exists()); + Ok(()) + } + + #[test] + fn rotate_without_backups_discards_instead_of_archiving() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let log = temp.path().join("app.log"); + + fs::write(&log, b"discarded\n")?; + let rotated = rotate(&log, 0).expect("rotated"); + + assert_eq!(rotated.bytes, 10); + assert!(rotated.archived.is_none()); + assert_eq!(fs::read(&log)?.len(), 0); + assert!(!segment_path(&log, 1).exists()); + Ok(()) + } + + #[test] + fn rotate_if_oversized_respects_the_cap() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let log = temp.path().join("app.log"); + + fs::write(&log, vec![b'x'; 100])?; + assert!(rotate_if_oversized(&log, 4096, 3).is_none()); + assert_eq!(fs::read(&log)?.len(), 100); + + fs::write(&log, vec![b'x'; 8192])?; + assert!(rotate_if_oversized(&log, 4096, 3).is_some()); + assert_eq!(fs::read(&log)?.len(), 0); + assert_eq!(fs::read(segment_path(&log, 1))?.len(), 8192); + + // A zero cap disables rotation entirely. + fs::write(&log, vec![b'x'; 8192])?; + assert!(rotate_if_oversized(&log, 0, 3).is_none()); + assert_eq!(fs::read(&log)?.len(), 8192); + Ok(()) + } + + #[test] + fn truncate_is_unconditional_and_tolerates_a_missing_file() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let log = temp.path().join("app.log"); + + truncate(&log); + assert!(!log.exists(), "truncate must not create the file"); + + fs::write(&log, b"stale output\n")?; + truncate(&log); + assert_eq!(fs::read(&log)?.len(), 0); + Ok(()) + } + + #[test] + fn rotation_note_says_where_the_output_went() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let log = temp.path().join("serial.log"); + + fs::write(&log, vec![b'x'; 8192])?; + let rotated = rotate(&log, 3).expect("rotated"); + append_rotation_note(&log, &rotated); + + let note = String::from_utf8_lossy(&fs::read(&log)?).into_owned(); + assert!( + note.contains("rotated 8192 bytes to serial.log.1"), + "{note:?}" + ); + Ok(()) + } + + #[test] + fn rotation_note_does_not_claim_an_archive_that_was_discarded() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let log = temp.path().join("serial.log"); + + fs::write(&log, vec![b'x'; 8192])?; + let rotated = rotate(&log, 0).expect("rotated"); + append_rotation_note(&log, &rotated); + + let note = String::from_utf8_lossy(&fs::read(&log)?).into_owned(); + assert!(note.contains("discarded 8192 bytes"), "{note:?}"); + assert!(!note.contains("serial.log.1"), "{note:?}"); + Ok(()) + } +} diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index 8ab3be383..17b93d194 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -7,7 +7,8 @@ use std::{path::Path, time::Duration}; use anyhow::{anyhow, Context, Result}; use app::App; use clap::{Args as ClapArgs, Parser, Subcommand}; -use config::Config; +use config::{Config, NetdConfig}; +use dstack_api_auth::{Authenticator, HttpAuthConfig, HttpAuthFairing}; use guest_api_service::GuestApiHandler; use host_api_service::HostApiHandler; use main_service::RpcHandler; @@ -16,7 +17,6 @@ use rocket::{ fairing::AdHoc, figment::{providers::Serialized, Figment}, }; -use rocket_apitoken::ApiToken; use rocket_vsock_listener::VsockListener; use supervisor_client::SupervisorClient; use tracing::{error, info, warn}; @@ -24,22 +24,22 @@ use tracing::{error, info, warn}; mod app; mod config; mod discovery; +mod gpu_reset; mod guest_api_service; mod host_api_service; +mod logrotate; mod main_routes; mod main_service; +mod netd; mod one_shot; mod openapi; +mod vm_launcher; const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION"); -const GIT_REV: &str = git_version::git_version!( - args = ["--abbrev=20", "--always", "--dirty=-modified"], - prefix = "git:", - fallback = "unknown" -); +const GIT_REV: &str = dstack_build_info::git_revision!(); fn app_version() -> String { - format!("v{CARGO_PKG_VERSION} ({GIT_REV})") + dstack_build_info::app_version!() } #[derive(Parser)] @@ -48,6 +48,9 @@ struct Args { /// Path to the configuration file #[arg(short, long)] config: Option, + /// Override the netd socket used by the VMM (useful without systemd). + #[arg(long, global = true)] + netd_socket: Option, /// Subcommand to run #[command(subcommand)] command: Option, @@ -58,8 +61,22 @@ enum Command { /// Start the VMM server (default mode) #[default] Serve, + /// Validate the effective server configuration without starting services. + CheckConfig, /// One-shot VM execution mode for debugging Run(RunArgs), + /// Run the privileged TAP and libvirt nwfilter broker. + Netd(NetdArgs), + /// Internal per-VM QEMU/swtpm launcher. + #[command(hide = true)] + VmLauncher(VmLauncherArgs), +} + +#[derive(ClapArgs)] +struct NetdArgs { + /// Override the Unix socket configured in [netd]. + #[arg(long)] + socket: Option, } #[derive(ClapArgs)] @@ -74,20 +91,34 @@ struct RunArgs { dry_run: bool, } -async fn run_external_api(app: App, figment: Figment, api_auth: ApiToken) -> Result<()> { +#[derive(ClapArgs)] +struct VmLauncherArgs { + /// Path to the generated VM launch specification. + #[arg(long)] + spec: String, +} + +async fn run_external_api(app: App, figment: Figment, api_auth: Authenticator) -> Result<()> { let version = app_version(); let openapi_doc = openapi::build_openapi_doc(&version)?; let external_api = rocket::custom(figment) .mount("/", main_routes::routes()) .mount("/guest", ra_rpc::prpc_routes!(App, GuestApiHandler)) - .mount("/api", ra_rpc::prpc_routes!(App, HostApiHandler)) .mount( "/prpc", ra_rpc::prpc_routes!(App, RpcHandler, trim: "Teepod."), ) .manage(app) - .manage(api_auth) + .attach(HttpAuthFairing::new( + api_auth, + HttpAuthConfig { + realm: "dstack-vmm API".into(), + token_header: Some("X-Admin-Token".into()), + allow_get_query_token: true, + }, + )) + .mount("/", dstack_api_auth::routes()) .attach(AdHoc::on_response("Add app rev header", |_req, res| { Box::pin(async move { res.set_raw_header("X-App-Version", app_version()); @@ -137,11 +168,27 @@ async fn auto_restart_task(app: App) { let mut interval = tokio::time::interval(Duration::from_secs(app.config.cvm.auto_restart.interval)); loop { + interval.tick().await; info!("Checking for exited VMs"); if let Err(err) = app.try_restart_exited_vms().await { error!("Failed to restart exited VMs: {err:?}"); } + } +} + +async fn log_rotation_task(app: App) { + if app.config.cvm.log.max_bytes == 0 { + info!("Log rotation is disabled"); + return; + } + let mut interval = tokio::time::interval(Duration::from_secs( + app.config.cvm.log.check_interval_secs.max(1), + )); + loop { interval.tick().await; + if let Err(err) = app.rotate_oversized_logs().await { + error!("Failed to rotate logs: {err:?}"); + } } } @@ -155,17 +202,75 @@ async fn main() -> Result<()> { let args = Args::parse(); + // The per-VM launcher must stay minimal: do not load or validate the VMM + // server configuration in this mode. + if let Some(Command::VmLauncher(launcher_args)) = &args.command { + return vm_launcher::run(Path::new(&launcher_args.spec)).await; + } + let figment = config::load_config_figment(args.config.as_deref()); - let config = Config::extract_or_default(&figment)?.abs_path()?; + if let Some(Command::Netd(netd_args)) = &args.command { + let mut netd_config: NetdConfig = figment + .extract_inner("netd") + .context("failed to load [netd] configuration")?; + if let Some(socket) = args.netd_socket.as_deref() { + netd_config.socket = socket.into(); + } + if let Some(socket) = netd_args.socket.as_deref() { + netd_config.socket = socket.into(); + } + if netd_config.network_filter.is_none() { + // netd and the VMM normally share one vmm.toml, so the node has + // already stated whether its bridge traffic is filtered and with + // what. Reading it here keeps the two from drifting apart, which is + // what a second setting to keep in sync would invite. + // + // A malformed section is an error rather than a default: this is + // the daemon's security policy, and `[cvm.network_filter] mode = + // "Libvirt"` -- which the VMM itself refuses to start on -- must not + // quietly resolve to "filter nothing" here. + netd_config.network_filter = Some( + figment + .extract_inner("cvm.network_filter") + .context("failed to load [cvm.network_filter] for netd")?, + ); + } + return netd::serve(netd_config).await; + } - // Validate host API configuration + let mut config = Config::extract_or_default(&figment)?.abs_path()?; + config.cvm.instance_id = netd::instance_id(&config.cvm.instance_id, config.run_path.as_path()); + if let Some(socket) = args.netd_socket.as_deref() { + config.netd.socket = socket.into(); + } + + // Preserve the existing startup validation. The broader static checks are + // opt-in through `check-config` until they have seen wider deployment use. config .host_api .validate() .context("Invalid host_api configuration")?; + config + .cvm + .auto_restart + .validate() + .context("Invalid cvm.auto_restart configuration")?; // Handle commands match args.command.unwrap_or_default() { + Command::VmLauncher(_) => unreachable!("launcher mode handled before config loading"), + Command::CheckConfig => { + config.validate()?; + let _: rocket::listener::Endpoint = figment + .extract_inner("address") + .context("Invalid management API address")?; + let _: u16 = figment + .extract_inner("port") + .context("Invalid management API port")?; + println!("configuration is valid"); + return Ok(()); + } + Command::Netd(_) => unreachable!("netd mode handled before server startup"), Command::Run(run_args) => { // One-shot VM execution mode return one_shot::run_one_shot( @@ -183,6 +288,10 @@ async fn main() -> Result<()> { // Register this VMM instance for local discovery discovery::cleanup_stale_registrations(); + // whether the management API binds a TCP address reachable beyond the local + // host (i.e. not a Unix socket and not a loopback IP). Used to warn when the + // surface is exposed without authentication. + let mut listen_tcp_public = false; let listen_address = { // Use Rocket's Endpoint type to parse the address exactly as Rocket would, // then override the port with the figment's port value (matching Rocket's behavior). @@ -191,6 +300,7 @@ async fn main() -> Result<()> { match endpoint.tcp() { Some(addr) => { let port: u16 = figment.extract_inner("port").unwrap_or(addr.port()); + listen_tcp_public = !addr.ip().is_loopback(); format!("{}:{port}", addr.ip()) } None => endpoint.to_string(), @@ -211,7 +321,22 @@ async fn main() -> Result<()> { } }; - let api_auth = ApiToken::new(config.auth.tokens.clone(), config.auth.enabled); + let mut api_auth = if config.auth.enabled { + Authenticator::from_tokens(config.auth.tokens.clone()) + } else { + Authenticator::disabled() + }; + if config.auth.enabled && !config.auth.htpasswd_file.as_os_str().is_empty() { + api_auth = api_auth.with_htpasswd_file(&config.auth.htpasswd_file)?; + } + if !config.auth.enabled && listen_tcp_public { + warn!( + "the management API is bound to a non-loopback address ({listen_address}) with \ + `[auth] enabled = false`: the entire VMM control surface (create/stop VM, UI, \ + pRPC) is exposed WITHOUT authentication. set `[auth] enabled = true` with a \ + token, or bind `address` to localhost / a Unix socket." + ); + } let supervisor = { let cfg = &config.supervisor; let abs_exe = Path::new(&cfg.exe).absolutize()?; @@ -229,6 +354,7 @@ async fn main() -> Result<()> { let state = app::App::new(config, supervisor); state.reload_vms().await.context("Failed to reload VMs")?; tokio::spawn(auto_restart_task(state.clone())); + tokio::spawn(log_rotation_task(state.clone())); tokio::select! { result = run_external_api(state.clone(), figment.clone(), api_auth) => { diff --git a/dstack/vmm/src/main_routes.rs b/dstack/vmm/src/main_routes.rs index c74e7e63b..ad6219740 100644 --- a/dstack/vmm/src/main_routes.rs +++ b/dstack/vmm/src/main_routes.rs @@ -10,7 +10,6 @@ use rocket::{ response::{status::Custom, stream::TextStream}, routes, Route, State, }; -use rocket_apitoken::Authorized; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; use tokio::time::timeout; @@ -57,11 +56,6 @@ async fn beta(app: &State) -> (ContentType, String) { index(app).await } -#[get("/v0")] -async fn v0(app: &State) -> (ContentType, String) { - render_console(file_or_include_str!("console_v0.html"), app) -} - #[get("/res/")] async fn res(path: &str) -> Result<(ContentType, String), Custom> { match path { @@ -106,27 +100,37 @@ impl Drop for StreamCounter { #[get("/logs?&&&&")] fn vm_logs( - _auth: Authorized, app: &State, id: String, follow: bool, ansi: bool, lines: Option, ch: Option<&str>, -) -> TextStream![String] { - let workdir = app.work_dir(&id); - let ch = ch.unwrap_or("serial").to_string(); - TextStream! { - let log_file = match ch.as_str() { - "serial" => workdir.serial_file(), - "stdout" => workdir.stdout_file(), - "stderr" => workdir.stderr_file(), - _ => { - yield format!("Unknown channel {ch}"); - return; - } - }; +) -> Result> { + // Resolve only an inventory-owned VM before deriving a filesystem path. + // This keeps arbitrary IDs (including traversal strings) outside run_path. + if app.lock().get(&id).is_none() { + return Err(Custom( + rocket::http::Status::NotFound, + "VM not found".to_string(), + )); + } + let workdir = app + .work_dir(&id) + .map_err(|err| Custom(rocket::http::Status::BadRequest, err.to_string()))?; + let log_file = match ch.unwrap_or("serial") { + "serial" => workdir.serial_file(), + "stdout" => workdir.stdout_file(), + "stderr" => workdir.stderr_file(), + channel => { + return Err(Custom( + rocket::http::Status::BadRequest, + format!("Unknown channel {channel}"), + )); + } + }; + Ok(TextStream! { let counter = StreamCounter::new(); const DEFAULT_TAIL_LINES: usize = 10000; @@ -170,18 +174,16 @@ fn vm_logs( yield strip_ansi_escapes::strip_str(&line_str); } } - Ok(None) => { - break; - } + Ok(None) => break, Err(err) => { yield format!(""); break; } } } - } + }) } pub fn routes() -> Vec { - routes![index, v1, beta, v0, res, vm_logs] + routes![index, v1, beta, res, vm_logs] } diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 932c4c17d..7d0c7099b 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -2,7 +2,9 @@ // // SPDX-License-Identifier: Apache-2.0 +use std::collections::HashSet; use std::ops::Deref; +use std::path::{Component, Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{bail, Context, Result}; @@ -10,23 +12,24 @@ use dstack_types::AppCompose; use dstack_vmm_rpc as rpc; use dstack_vmm_rpc::vmm_server::{VmmRpc, VmmServer}; use dstack_vmm_rpc::{ - AppId, ComposeHash as RpcComposeHash, DhcpLeaseRequest, GatewaySettings, GetInfoResponse, - GetMetaResponse, Id, ImageInfo as RpcImageInfo, ImageListResponse, KmsSettings, - ListGpusResponse, PublicKeyResponse, PullRegistryImageRequest, RegistryImageInfo, - RegistryImageListResponse, ReloadVmsResponse, ResizeVmRequest, ResourcesSettings, - StatusRequest, StatusResponse, SvListResponse, SvProcessInfo, UpdateVmRequest, VersionResponse, - VmConfiguration, + AppId, ComposeHash as RpcComposeHash, GatewaySettings, GetInfoResponse, GetMetaResponse, Id, + ImageInfo as RpcImageInfo, ImageListResponse, KmsSettings, ListGpusResponse, PublicKeyResponse, + PullRegistryImageRequest, RegistryImageInfo, RegistryImageListResponse, ReloadVmsResponse, + ResizeVmRequest, ResourcesSettings, StatusRequest, StatusResponse, SvListResponse, + SvProcessInfo, UpdateVmRequest, VersionResponse, VmConfiguration, }; use fs_err as fs; use or_panic::ResultOrPanic; +use path_absolutize::Absolutize; use ra_rpc::{CallContext, RpcCall}; use tracing::{info, warn}; use crate::app::{ - resolve_networking, validate_resolved_network, validate_resolved_networks, App, AttachMode, - GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir, + mode_carries_ingress, needs_swtpm, resolve_networking, validate_resolved_network, + validate_resolved_networks, App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, + VmWorkDir, }; -use crate::config::{CvmConfig, Networking, NetworkingMode}; +use crate::config::{CvmConfig, Networking, NetworkingMode, NicNetworking}; fn hex_sha256(data: &str) -> String { use sha2::Digest; @@ -58,6 +61,29 @@ fn app_id_of(compose_file: &str) -> String { truncate40(&hex_sha256(compose_file)).to_string() } +fn key_provider_from_compose(compose_file: &str) -> Result { + let compose: serde_json::Value = + serde_json::from_str(compose_file).context("invalid app compose JSON")?; + if let Some(provider) = compose.get("key_provider").filter(|value| !value.is_null()) { + return serde_json::from_value(provider.clone()).context("invalid key_provider"); + } + if compose + .get("kms_enabled") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + { + return Ok(dstack_types::KeyProviderKind::Kms); + } + if compose + .get("local_key_provider_enabled") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + { + return Ok(dstack_types::KeyProviderKind::Local); + } + Ok(dstack_types::KeyProviderKind::None) +} + /// Validate the VM label, restricting it to a safe character set to prevent injection vectors. fn validate_label(label: &str) -> Result<()> { fn is_valid_label_char(c: char) -> bool { @@ -135,6 +161,87 @@ pub fn resolve_gpus(gpu_cfg: &rpc::GpuConfig) -> Result { } } +fn port_mappings_conflict(left: &PortMapping, right: &PortMapping) -> bool { + left.protocol.as_str() == right.protocol.as_str() + && left.from == right.from + && (left.address == right.address + || left.address.is_unspecified() + || right.address.is_unspecified()) +} + +/// The backend each of a VM's NICs resolves to, with an empty list standing for +/// the node default's single NIC. +fn resolved_nic_modes( + networks: &[NicNetworking], + cvm_config: &CvmConfig, + vcpu: u32, +) -> Vec { + let node_default = [cvm_config.networking.nic.clone()]; + let requested = if networks.is_empty() { + &node_default[..] + } else { + networks + }; + requested + .iter() + .map(|networking| resolve_networking(networking, cvm_config, vcpu).nic.mode) + .collect() +} + +/// Rejects a mapping pinned to a NIC that cannot carry it. +/// +/// Both ways of getting that wrong are refused here, at deployment, where the +/// caller is present to be told: a NIC the VM does not have, and one whose +/// backend has no mechanism to publish a host port. Macvtap bypasses the host +/// bridge and a custom netdev is a string the VMM does not interpret, so a +/// mapping pinned to either used to resolve to that NIC and then fall out of +/// every branch that could act on it -- no `hostfwd=`, no netd request, and no +/// warning either. +fn validate_port_mapping_nics(mappings: &[PortMapping], modes: &[NetworkingMode]) -> Result<()> { + let nic_count = modes.len(); + for mapping in mappings { + let Some(index) = mapping.nic_index else { + continue; + }; + let Some(mode) = modes.get(index) else { + bail!( + "port mapping {} {}:{} names NIC {index}, but this VM has {nic_count}", + mapping.protocol.as_str(), + mapping.address, + mapping.from + ); + }; + if !mode_carries_ingress(*mode) { + bail!( + "port mapping {} {}:{} names NIC {index}, which is {} and cannot publish a host \ + port; use a user-mode or bridge NIC", + mapping.protocol.as_str(), + mapping.address, + mapping.from, + mode.as_str(), + ); + } + } + Ok(()) +} + +fn validate_unique_port_mappings(mappings: &[PortMapping]) -> Result<()> { + for (index, mapping) in mappings.iter().enumerate() { + if mappings[..index] + .iter() + .any(|other| port_mappings_conflict(mapping, other)) + { + bail!( + "duplicate host port mapping: {} {}:{}", + mapping.protocol.as_str(), + mapping.address, + mapping.from + ); + } + } + Ok(()) +} + // Shared function to create manifest from VM configuration pub fn create_manifest_from_vm_config( request: VmConfiguration, @@ -166,9 +273,16 @@ pub fn create_manifest_from_vm_config( protocol, from, to, + nic_index: p.nic_index.map(|index| index as usize), }) }) .collect::>>()?; + validate_unique_port_mappings(&port_map)?; + let networks = networks_from_vm_config(&request, cvm_config)?; + validate_port_mapping_nics( + &port_map, + &resolved_nic_modes(&networks, cvm_config, request.vcpu), + )?; let app_id = match &request.app_id { Some(id) => id.strip_prefix("0x").unwrap_or(id).to_lowercase(), @@ -183,6 +297,21 @@ pub fn create_manifest_from_vm_config( Some(gpus) => resolve_gpus_with_config(gpus, cvm_config)?, None => GpuConfig::default(), }; + let verity_volumes = extract_verity_volumes(&request.compose_file)?; + dstack_types::validate_verity_volumes(&verity_volumes).map_err(anyhow::Error::msg)?; + let volumes = resolve_volumes(&verity_volumes, cvm_config)?; + + let simulated_tee = request + .simulated_tee + .as_deref() + .map(str::parse) + .transpose() + .map_err(anyhow::Error::msg)?; + if simulated_tee.is_some() && cvm_config.tee_simulator.is_none() { + bail!("tee simulator credentials are not configured on this VMM"); + } + let key_provider = key_provider_from_compose(&request.compose_file)?; + let swtpm = needs_swtpm(key_provider, simulated_tee); Ok(Manifest { id, @@ -199,42 +328,291 @@ pub fn create_manifest_from_vm_config( gpus: Some(gpus), kms_urls: request.kms_urls.clone(), gateway_urls: request.gateway_urls.clone(), - no_tee: request.no_tee, - networks: networks_from_vm_config(&request, cvm_config)?, + no_tee: request.no_tee || simulated_tee.is_some(), + simulated_tee, + swtpm, + networks, + volumes, }) } -fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Result> { +/// Extract only the field understood by this VMM. Keep every other app-compose +/// field opaque so newer guest schemas and legacy third-party clients remain +/// compatible with older VMMs. +fn extract_verity_volumes(compose: &str) -> Result> { + let Ok(compose) = serde_json::from_str::(compose) else { + return Ok(vec![]); + }; + let Some(volumes) = compose.get("verity_volumes") else { + return Ok(vec![]); + }; + serde_json::from_value(volumes.clone()).context("invalid verity_volumes in app-compose") +} + +/// Resolve requested volumes against `cvm.volumes_dir`. Each `source` must be a +/// bare file name under that directory; the host attaches the bytes, and the +/// guest verifies content against the measured `verity_root`. +fn resolve_volumes( + reqs: &[dstack_types::VerityVolume], + cvm_config: &crate::config::CvmConfig, +) -> Result> { + if reqs.is_empty() { + return Ok(vec![]); + } + let dir = cvm_config.volumes_dir.trim(); + if dir.is_empty() { + bail!("volumes requested but cvm.volumes_dir is not configured"); + } + let base = fs::canonicalize(dir)?; + let mut roots = HashSet::new(); + reqs.iter() + .filter(|volume| { + let first = roots.insert(volume.verity_root); + if !first { + warn!( + root = %hex::encode(volume.verity_root), + source = volume.source, + "not attaching duplicate verity root" + ); + } + first + }) + .map(|v| { + let real = resolve_volume_source(&base, &v.source)?; + Ok(crate::app::VmVolume { + source: real.to_string_lossy().into_owned(), + }) + }) + .collect() +} + +fn resolve_volume_source(base: &Path, source: &str) -> Result { + if source.is_empty() { + bail!("invalid volume source: empty path"); + } + + let source_path = Path::new(source); + let mut components = source_path.components(); + if !matches!(components.next(), Some(Component::Normal(_))) || components.next().is_some() { + bail!("invalid volume source '{source}': must be a bare file name"); + } + + let real = fs::canonicalize(base.join(source_path))?; + real.absolutize_virtually(base) + .with_context(|| format!("volume '{source}' escapes volumes_dir"))?; + + // QEMU's -drive parser treats ',' as an option separator and '=' as an + // option key/value delimiter. Keep this guard while volumes are attached + // through `-drive file=...`. + let real_str = real.to_string_lossy(); + if real_str.contains(',') || real_str.contains('=') { + bail!("volume '{source}' resolves to a path with ',' or '='"); + } + + Ok(real) +} + +fn networking_from_proto( + proto: &rpc::NetworkingConfig, + cvm_config: &CvmConfig, +) -> Result> { let bridge = proto.bridge_name.trim().to_string(); - let mode = match proto.mode.as_str() { - "bridge" => NetworkingMode::Bridge, - "user" => NetworkingMode::User, - "" if bridge.is_empty() => return Ok(None), - "" => bail!("networking mode is required when bridge is set"), + let parent = proto.parent.trim().to_string(); + // Checked before anything reads `queues`, because the "no override at all" + // arm below returns early: a request of `{queues: 0}` and nothing else + // would otherwise be answered with the vCPU-scaled default -- up to + // sixteen queue pairs -- for a caller who asked for none. + if proto.queues == Some(0) { + bail!("networking queues must be at least 1, or unset to follow the vCPU count"); + } + let tuned = proto.vhost.is_some() || proto.queues.is_some(); + // Naming a bridge or a macvtap parent is naming a backend, and a backend + // needs a mode to go with it. Without one the entry would freeze whichever + // mode the node happened to have, then report a mode-less override still + // carrying a field only one mode accepts -- something nothing can send + // back once the node moves on. + let names_backend = !bridge.is_empty() || !parent.is_empty(); + // A request that only tunes the data plane keeps the node's backend. Node + // policy governs which backend a caller may *choose*, so inheriting one + // must not be denied by it. + let (mode, chosen) = match proto.mode.as_str() { + "bridge" => (NetworkingMode::Bridge, true), + "user" => (NetworkingMode::User, true), + "macvtap" => (NetworkingMode::Macvtap, true), + "" if !names_backend && !tuned => return Ok(None), + "" if !names_backend => (cvm_config.networking.nic.mode, false), + "" => bail!("networking mode is required when a bridge or macvtap parent is set"), "custom" => bail!("custom networking mode is manifest-only"), other => bail!("unsupported networking mode '{other}'"), }; + if chosen && !cvm_config.allowed_network_modes.contains(&mode) { + bail!( + "networking mode '{}' is not allowed by node policy", + proto.mode + ); + } if mode != NetworkingMode::Bridge && !bridge.is_empty() { bail!("bridge_name is only valid for bridge networking mode"); } - Ok(Some(Networking { + if mode != NetworkingMode::Macvtap && !parent.is_empty() { + bail!("parent is only valid for macvtap networking mode"); + } + // `GetInfo` reports the resolved node defaults, and both vmm-cli and the + // web UI read that, change one field, and send the rest back. Naming a + // value the node would have supplied anyway therefore has to be accepted: + // leaving the field empty already yields exactly it, so echoing it grants + // nothing that policy was withholding. + // Node values are compared trimmed, the way the request's are: node + // configuration validation tolerates surrounding whitespace, and a value + // resolution would supply must not become unsendable over a space. + let node = &cvm_config.networking; + let macvtap_mode = proto.macvtap_mode.trim(); + if !macvtap_mode.is_empty() && macvtap_mode != node.macvtap_mode.trim() { + bail!("macvtap_mode is node-controlled and cannot be set by deployment RPCs"); + } + if !bridge.is_empty() + && bridge != node.nic.bridge.trim() + && !cvm_config.allowed_bridges.contains(&bridge) + { + bail!("bridge_name '{bridge}' is not allowed by node policy"); + } + if !parent.is_empty() + && parent != node.nic.parent.trim() + && !cvm_config.allowed_macvtap_parents.contains(&parent) + { + bail!("macvtap parent '{parent}' is not allowed by node policy"); + } + // Same rule as the queue count below: a backend the caller chose and that + // has no vhost data plane is a request they can fix, so say so. An + // inherited one is not, and reads as off until the node moves. + if chosen && proto.vhost == Some(true) && !Networking::mode_supports_vhost(mode) { + bail!("{} networking has no vhost data plane", proto.mode); + } + // Queue pairs cost a host vhost thread and a pair of MSI-X vectors each, so + // the node caps what a deployment may ask for. + // + let queues = proto.queues; + if let Some(queues) = queues { + if queues > cvm_config.max_net_queues { + bail!( + "networking queues must not exceed {} on this node", + cvm_config.max_net_queues + ); + } + // Only a backend the caller *chose* is theirs to be wrong about. One + // they inherited can change under them -- that is the point of + // inheriting -- and refusing the request afterwards would strand the + // VM: GetInfo would keep reporting a queue count that nothing is + // allowed to send back. `queue_pairs()` already reads as one on both + // of these, so the request simply lies dormant until the node moves to + // a backend that can honour it. + if chosen && queues > 1 && !Networking::mode_supports_multiqueue(mode) { + bail!("{} networking does not support multiple queues", proto.mode); + } + } + // Every field the node owns -- the macvtap forwarding mode, the MAC prefix, + // the user-mode network parameters, a custom netdev string -- is absent by + // construction now rather than by remembering to write `String::new()` for + // each of them. + Ok(Some(NicNetworking { mode, + // The caller named no backend, so the node keeps owning which one this + // NIC uses. `mode` above is only the node's current choice. + inherit_mode: !chosen, bridge, - mac_prefix: String::new(), - net: String::new(), - dhcp_start: String::new(), - restrict: false, - netdev: String::new(), - forward_service_enabled: false, + parent, + vhost: proto.vhost, + queues, })) } -fn network_from_required_proto(proto: &rpc::NetworkingConfig) -> Result { - networking_from_proto(proto)?.context("networking mode is required") +/// Networking modes a client should offer. +/// +/// A mode the host could serve but node policy forbids is not one of them: +/// offering it puts a choice in the deploy dialog whose only outcome is "not +/// allowed by node policy", with nothing for the operator to do about it. A VM +/// can still be given a data plane override without naming any mode, which is +/// how a node whose own backend is not caller-selectable stays tunable. +fn advertised_modes(cvm_config: &CvmConfig, host_can_bridge: bool) -> Vec { + [ + (NetworkingMode::User, "user", true), + (NetworkingMode::Bridge, "bridge", host_can_bridge), + (NetworkingMode::Macvtap, "macvtap", true), + ] + .into_iter() + .filter(|(mode, _, host_supports)| { + *host_supports && cvm_config.allowed_network_modes.contains(mode) + }) + .map(|(_, name, _)| name.to_string()) + .collect() +} + +/// The node's policy, widened by what this VM's own NICs already pin. +/// +/// Deployment allowlists govern what a caller may *newly* select. A value one +/// of this VM's interfaces is already running on was selected when it was +/// still allowed, and it is reported back on every `GetInfo`; refusing it +/// would strand the VM rather than withhold anything. +fn held_networking_config(cvm_config: &CvmConfig, held: &[NicNetworking]) -> CvmConfig { + let mut widened = cvm_config.clone(); + for networking in held { + // Only a field the entry's own mode owns. Resolution starts from the + // node's whole `[cvm.networking]` value, so a bridge entry also carries + // whatever macvtap parent the node happened to have configured, and a + // macvtap entry carries its bridge -- baggage the VM never used. Reading + // those as "already running on it" would widen policy from a value + // nothing ever attached to, and let a VM move to a backend the node + // forbids. + if networking.mode == NetworkingMode::Bridge && !networking.bridge.is_empty() { + widened.allowed_bridges.push(networking.bridge.clone()); + } + if networking.mode == NetworkingMode::Macvtap && !networking.parent.is_empty() { + widened + .allowed_macvtap_parents + .push(networking.parent.clone()); + } + // Same rule for the queue cap. Lowering `max_net_queues` bounds what a + // deployment may newly ask for; it does not retroactively re-tune VMs + // that are already pinned above it, and those keep reporting their + // count on every `GetInfo`. Without this, lowering the cap makes every + // networking update on such a VM fail on a field the operator never + // typed -- including the `--net-queues auto` that would unpin it. + if let Some(queues) = networking.queues { + widened.max_net_queues = widened.max_net_queues.max(queues); + } + } + widened +} + +/// A NIC that overrides nothing: whatever the node's `[cvm.networking]` says, +/// now and after the operator changes it. +fn node_default_networking(cvm_config: &CvmConfig) -> NicNetworking { + NicNetworking { + mode: cvm_config.networking.nic.mode, + inherit_mode: true, + ..NicNetworking::default() + } +} + +fn network_from_required_proto( + proto: &rpc::NetworkingConfig, + cvm_config: &CvmConfig, +) -> Result { + // An entry in a list that overrides nothing is not a missing mode: it is a + // NIC that follows the node entirely. Only the singular `networking` field + // can mean "no override at all", because there the absence is the message. + Ok(networking_from_proto(proto, cvm_config)? + .unwrap_or_else(|| node_default_networking(cvm_config))) } -fn networks_from_proto(networks: &[rpc::NetworkingConfig]) -> Result> { - networks.iter().map(network_from_required_proto).collect() +fn networks_from_proto( + networks: &[rpc::NetworkingConfig], + cvm_config: &CvmConfig, +) -> Result> { + networks + .iter() + .map(|network| network_from_required_proto(network, cvm_config)) + .collect() } fn validate_default_network(cvm_config: &CvmConfig) -> Result<()> { @@ -242,15 +620,68 @@ fn validate_default_network(cvm_config: &CvmConfig) -> Result<()> { } fn resolve_requested_networks( - networks: &[Networking], + requests: &[NicNetworking], cvm_config: &CvmConfig, -) -> Result> { - let resolved = networks + vcpu: u32, +) -> Result> { + let merged = requests .iter() - .map(|networking| resolve_networking(networking, cvm_config)) + .map(|request| resolve_networking(request, cvm_config, vcpu)) .collect::>(); - validate_resolved_networks(&resolved)?; - Ok(resolved) + // Validate the merged view, because that is what the launch sees, then + // record the narrower view the manifest keeps. + validate_resolved_networks(&merged)?; + Ok(manifest_networks(merged, requests)) +} + +/// What a deployment records against the VM, given the merged view its launch +/// would see. +/// +/// The backend a deployment *chose* is pinned here -- its mode, and the bridge +/// or macvtap parent that names it -- so a VM keeps the segment it was put on +/// for life. Everything else stays owned by the node and is re-read at every +/// launch: the MAC prefix, the user-mode subnet and DHCP start, the macvtap +/// forwarding mode, and the data plane an operator may want to roll back +/// node-wide. A VM's own record cannot carry any of those. +fn manifest_networks(merged: Vec, requests: &[NicNetworking]) -> Vec { + merged + .into_iter() + .zip(requests) + .map(|(entry, request)| { + if request.inherit_mode { + // The caller named no backend, so none of the node's matching + // identity fields are theirs to keep. + return request.clone(); + } + // Taking `entry.nic` rather than the whole resolved value is what + // keeps the node's half out of the VM's record; it used to take the + // whole thing and then clear the one node field anybody noticed. + // + // The two identity fields still share one type, and resolution + // fills both from the node, so scope them here -- once, where the + // record is written -- rather than leaving every later reader to + // remember which one its mode owns. Two already have to. + let mode = entry.nic.mode; + NicNetworking { + bridge: if mode == NetworkingMode::Bridge { + entry.nic.bridge + } else { + String::new() + }, + parent: if mode == NetworkingMode::Macvtap { + entry.nic.parent + } else { + String::new() + }, + // Data plane tuning is not identity: leave what the caller did + // not ask for unset. + vhost: request.vhost, + queues: request.queues, + mode, + inherit_mode: entry.nic.inherit_mode, + } + }) + .collect() } fn has_host_bridge_interface() -> bool { @@ -265,13 +696,13 @@ fn has_host_bridge_interface() -> bool { fn networks_from_vm_config( request: &VmConfiguration, cvm_config: &CvmConfig, -) -> Result> { +) -> Result> { if !request.networks.is_empty() { - let networks = networks_from_proto(&request.networks)?; - resolve_requested_networks(&networks, cvm_config) + let networks = networks_from_proto(&request.networks, cvm_config)?; + resolve_requested_networks(&networks, cvm_config, request.vcpu) } else if let Some(networking) = request.networking.as_ref() { - match networking_from_proto(networking)? { - Some(networking) => resolve_requested_networks(&[networking], cvm_config), + match networking_from_proto(networking, cvm_config)? { + Some(networking) => resolve_requested_networks(&[networking], cvm_config, request.vcpu), None => Ok(vec![]), } } else { @@ -279,7 +710,62 @@ fn networks_from_vm_config( } } +fn validate_resize_request(request: &ResizeVmRequest) -> Result<()> { + if request.vcpu.is_none() + && request.memory.is_none() + && request.disk_size.is_none() + && request.image.is_none() + { + bail!("resize request contains no updates"); + } + if request.vcpu == Some(0) { + bail!("vcpu must be greater than zero"); + } + if request.memory == Some(0) { + bail!("memory must be greater than zero"); + } + if request.disk_size == Some(0) { + bail!("disk_size must be greater than zero"); + } + if request.image.as_deref() == Some("") { + bail!("image must not be empty"); + } + Ok(()) +} + impl RpcHandler { + fn validate_port_mapping_conflicts( + &self, + vm_id: Option<&str>, + mappings: &[PortMapping], + ) -> Result<()> { + validate_unique_port_mappings(mappings)?; + let state = self.app.lock(); + for vm in state.iter_vms() { + if vm_id == Some(vm.config.manifest.id.as_str()) { + continue; + } + for mapping in mappings { + if vm + .config + .manifest + .port_map + .iter() + .any(|existing| port_mappings_conflict(mapping, existing)) + { + bail!( + "host port mapping conflicts with VM {}: {} {}:{}", + vm.config.manifest.id, + mapping.protocol.as_str(), + mapping.address, + mapping.from + ); + } + } + } + Ok(()) + } + fn resolve_gpus(&self, gpu_cfg: &rpc::GpuConfig) -> Result { resolve_gpus_with_config(gpu_cfg, &self.app.config.cvm) } @@ -319,20 +805,27 @@ impl RpcHandler { if disk_size < manifest.disk_size { bail!("Cannot shrink disk size"); } - manifest.disk_size = disk_size; - - info!("Resizing disk to {}GB", disk_size); - let hda_path = vm_work_dir.hda_path(); - let new_size_str = format!("{}G", disk_size); - let output = std::process::Command::new("qemu-img") - .args(["resize", &hda_path.display().to_string(), &new_size_str]) - .output() - .context("Failed to resize disk")?; - if !output.status.success() { - bail!( - "Failed to resize disk: {}", - String::from_utf8_lossy(&output.stderr) - ); + if disk_size > manifest.disk_size { + let hda_path = vm_work_dir.hda_path(); + if hda_path.exists() { + info!("Resizing disk to {}GB", disk_size); + let new_size_str = format!("{}G", disk_size); + let output = std::process::Command::new("qemu-img") + .args(["resize", &hda_path.display().to_string(), &new_size_str]) + .output() + .context("Failed to resize disk")?; + if !output.status.success() { + bail!( + "Failed to resize disk: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + } else { + // A never-started stopped VM has no data disk yet. Its + // first launch creates hda.img from manifest.disk_size. + info!("Recording {}GB disk size for uninitialized VM", disk_size); + } + manifest.disk_size = disk_size; } } @@ -343,9 +836,11 @@ impl RpcHandler { impl VmmRpc for RpcHandler { async fn create_vm(self, request: VmConfiguration) -> Result { let manifest = create_manifest_from_vm_config(request.clone(), &self.app.config.cvm)?; + self.validate_port_mapping_conflicts(None, &manifest.port_map)?; let id = manifest.id.clone(); + info!(vm_id = %id, "create_vm RPC called"); let app_id = manifest.app_id.clone(); - let vm_work_dir = self.app.work_dir(&id); + let vm_work_dir = self.app.work_dir(&id)?; vm_work_dir .put_manifest(&manifest) .context("Failed to write manifest")?; @@ -380,6 +875,7 @@ impl VmmRpc for RpcHandler { } async fn start_vm(self, request: Id) -> Result<()> { + info!(vm_id = %request.id, "start_vm RPC called"); self.app .start_vm(&request.id) .await @@ -388,6 +884,7 @@ impl VmmRpc for RpcHandler { } async fn stop_vm(self, request: Id) -> Result<()> { + info!(vm_id = %request.id, "stop_vm RPC called"); self.app .stop_vm(&request.id) .await @@ -396,6 +893,7 @@ impl VmmRpc for RpcHandler { } async fn remove_vm(self, request: Id) -> Result<()> { + info!(vm_id = %request.id, "remove_vm RPC called"); self.app .remove_vm(&request.id) .await @@ -424,15 +922,17 @@ impl VmmRpc for RpcHandler { } async fn upgrade_app(self, request: UpdateVmRequest) -> Result { + info!(vm_id = %request.id, "upgrade_app RPC called"); self.update_vm(request).await } async fn update_vm(self, request: UpdateVmRequest) -> Result { + info!(vm_id = %request.id, "update_vm RPC called"); let new_id = if !request.compose_file.is_empty() { // check the compose file is valid let _app_compose: AppCompose = serde_json::from_str(&request.compose_file).context("Invalid compose file")?; - let compose_file_path = self.compose_file_path(&request.id); + let compose_file_path = self.compose_file_path(&request.id)?; if !compose_file_path.exists() { bail!("The instance {} not found", request.id); } @@ -444,16 +944,16 @@ impl VmmRpc for RpcHandler { Default::default() }; if !request.encrypted_env.is_empty() { - let encrypted_env_path = self.encrypted_env_path(&request.id); + let encrypted_env_path = self.encrypted_env_path(&request.id)?; fs::write(encrypted_env_path, &request.encrypted_env) .context("Failed to write encrypted env")?; } if !request.user_config.is_empty() { - let user_config_path = self.user_config_path(&request.id); + let user_config_path = self.user_config_path(&request.id)?; fs::write(user_config_path, &request.user_config) .context("Failed to write user config")?; } - let vm_work_dir = self.app.work_dir(&request.id); + let vm_work_dir = self.app.work_dir(&request.id)?; let mut manifest = vm_work_dir.manifest().context("Failed to read manifest")?; self.apply_resource_updates( &request.id, @@ -472,7 +972,7 @@ impl VmmRpc for RpcHandler { manifest.no_tee = no_tee; } if request.update_ports { - manifest.port_map = request + let port_map = request .ports .iter() .map(|p| { @@ -481,9 +981,12 @@ impl VmmRpc for RpcHandler { protocol: p.protocol.parse().context("Invalid protocol")?, from: p.host_port.try_into().context("Invalid host port")?, to: p.vm_port.try_into().context("Invalid vm port")?, + nic_index: p.nic_index.map(|index| index as usize), }) }) .collect::>>()?; + self.validate_port_mapping_conflicts(Some(&request.id), &port_map)?; + manifest.port_map = port_map; } if request.update_kms_urls { manifest.kms_urls = request.kms_urls.clone(); @@ -492,12 +995,18 @@ impl VmmRpc for RpcHandler { manifest.gateway_urls = request.gateway_urls.clone(); } if request.update_networking { - manifest.networks = if request.networks.is_empty() { + let networks = if request.networks.is_empty() { validate_default_network(&self.app.config.cvm)?; vec![] } else { - let networks = networks_from_proto(&request.networks)?; - resolve_requested_networks(&networks, &self.app.config.cvm)? + // A bridge or parent this VM already holds is not a new grant. + // `GetInfo` keeps reporting it, and read-modify-write keeps + // sending it back, so refusing it once the node changes its own + // default would make the VM's configuration unsendable -- with + // no flag anywhere to clear a field the caller never typed. + let cvm = held_networking_config(&self.app.config.cvm, &manifest.networks); + let networks = networks_from_proto(&request.networks, &cvm)?; + resolve_requested_networks(&networks, &cvm, manifest.vcpu)? }; let is_running = self .app @@ -506,9 +1015,34 @@ impl VmmRpc for RpcHandler { .await? .is_some_and(|info| info.state.status.is_running()); if !is_running { + let runtime_networks = vm_work_dir.runtime_networks(); + self.app + .remove_netd_networks(&request.id, &runtime_networks) + .await + .context("failed to remove previous netd-managed networking")?; vm_work_dir.clear_runtime_networks()?; } + manifest.networks = networks; + } + // Both only when this request moved one of the two halves, and after + // both, since either half can move and the other still has to agree + // with it. A VM deployed before the node could answer for its ports + // must stay editable in every other respect: read-modify-write sends + // the whole configuration back, and refusing a memory change over a + // port mapping nobody touched -- or over a node default that changed + // under it -- would make the VM unmanageable rather than fixed. + if request.update_ports || request.update_networking { + validate_port_mapping_nics( + &manifest.port_map, + &resolved_nic_modes(&manifest.networks, &self.app.config.cvm, manifest.vcpu), + )?; } + let compose_file = fs::read_to_string(vm_work_dir.app_compose_path()) + .context("failed to read app compose for swtpm decision")?; + manifest.swtpm = needs_swtpm( + key_provider_from_compose(&compose_file)?, + manifest.simulated_tee, + ); vm_work_dir .put_manifest(&manifest) .context("Failed to put manifest")?; @@ -517,9 +1051,6 @@ impl VmmRpc for RpcHandler { .load_vm(&vm_work_dir, &Default::default(), false) .await .context("Failed to load VM")?; - if request.update_ports { - self.app.reconfigure_port_forward(&request.id).await; - } Ok(Id { id: new_id }) } @@ -539,6 +1070,7 @@ impl VmmRpc for RpcHandler { } async fn get_info(self, request: Id) -> Result { + info!(vm_id = %request.id, "get_info RPC called"); if let Some(vm) = self.app.vm_info(&request.id).await? { Ok(GetInfoResponse { found: true, @@ -552,10 +1084,17 @@ impl VmmRpc for RpcHandler { } } - #[tracing::instrument(skip(self, request), fields(id = request.id))] async fn resize_vm(self, request: ResizeVmRequest) -> Result<()> { - info!("Resizing VM: {:?}", request); - let vm_work_dir = self.app.work_dir(&request.id); + info!( + vm_id = %request.id, + vcpu = ?request.vcpu, + memory = ?request.memory, + disk_size = ?request.disk_size, + image = ?request.image, + "resize_vm RPC called" + ); + validate_resize_request(&request)?; + let vm_work_dir = self.app.work_dir(&request.id)?; let mut manifest = vm_work_dir.manifest().context("failed to read manifest")?; self.apply_resource_updates( &request.id, @@ -578,6 +1117,7 @@ impl VmmRpc for RpcHandler { } async fn shutdown_vm(self, request: Id) -> Result<()> { + info!(vm_id = %request.id, "shutdown_vm RPC called"); self.guest_agent_client(&request.id)?.shutdown().await?; Ok(()) } @@ -590,13 +1130,12 @@ impl VmmRpc for RpcHandler { } async fn get_meta(self) -> Result { - let mut supported_modes = vec!["user".to_string()]; let default_networking = &self.app.config.cvm.networking; let mut bridge_networking = default_networking.clone(); - bridge_networking.mode = NetworkingMode::Bridge; - if validate_resolved_network(&bridge_networking).is_ok() || has_host_bridge_interface() { - supported_modes.push("bridge".to_string()); - } + bridge_networking.nic.mode = NetworkingMode::Bridge; + let host_can_bridge = + validate_resolved_network(&bridge_networking).is_ok() || has_host_bridge_interface(); + let supported_modes = advertised_modes(&self.app.config.cvm, host_can_bridge); Ok(GetMetaResponse { kms: Some(KmsSettings { url: self @@ -630,13 +1169,15 @@ impl VmmRpc for RpcHandler { }), networking: Some(rpc::NetworkingCapabilities { supported_modes, - default_mode: match default_networking.mode { + default_mode: match default_networking.nic.mode { NetworkingMode::User => "user".to_string(), NetworkingMode::Bridge => "bridge".to_string(), NetworkingMode::Custom => String::new(), + NetworkingMode::Macvtap => "macvtap".to_string(), }, - forward_service_enabled: default_networking.forward_service_enabled, - default_bridge: default_networking.bridge.clone(), + default_bridge: default_networking.nic.bridge.clone(), + max_queues: self.app.config.cvm.max_net_queues, + default_vhost: default_networking.vhost_enabled(), }), }) } @@ -664,11 +1205,6 @@ impl VmmRpc for RpcHandler { self.app.reload_vms_sync().await } - async fn report_dhcp_lease(self, request: DhcpLeaseRequest) -> Result<()> { - self.app.report_dhcp_lease(&request.mac, &request.ip).await; - Ok(()) - } - async fn sv_list(self) -> Result { use supervisor_client::supervisor::ProcessStatus; let list = self.app.supervisor.list().await?; @@ -695,11 +1231,21 @@ impl VmmRpc for RpcHandler { } async fn sv_stop(self, request: Id) -> Result<()> { - self.app.supervisor.stop(&request.id).await?; - Ok(()) + info!(vm_id = %request.id, "sv_stop RPC called"); + // VM launcher processes own QEMU and swtpm children. Route them through + // the VM-aware stop path so the launcher can reap those children; the + // same helper preserves generic Supervisor stop semantics for every + // other process type. + self.app + .supervisor + .info(&request.id) + .await? + .context("Supervisor process not found")?; + self.app.stop_vm_process(&request.id).await } async fn sv_remove(self, request: Id) -> Result<()> { + info!(vm_id = %request.id, "sv_remove RPC called"); self.app.supervisor.remove(&request.id).await?; Ok(()) } @@ -869,11 +1415,73 @@ mod tests { gateway_urls: vec![], stopped: false, no_tee: false, + simulated_tee: None, networking: None, networks: vec![], } } + fn pinned(nic_index: Option) -> PortMapping { + PortMapping { + address: "0.0.0.0".parse().unwrap(), + protocol: crate::config::Protocol::Tcp, + from: 443, + to: 443, + nic_index, + } + } + + /// Both ways of naming a NIC that cannot publish a port are refused where + /// the caller is present to be told. A pin to macvtap or to a custom + /// netdev used to resolve to that NIC and then fall out of every branch + /// that could act on it, so the port simply never appeared. + #[test] + fn a_pin_is_checked_against_the_backend_and_not_only_the_count() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.parent = "eth0".into(); + cvm.networking.nic.bridge = "br0".into(); + + let bridge_then_macvtap = vec![ + NicNetworking { + mode: NetworkingMode::Bridge, + bridge: "br0".into(), + ..NicNetworking::default() + }, + NicNetworking { + mode: NetworkingMode::Macvtap, + parent: "eth0".into(), + ..NicNetworking::default() + }, + ]; + let modes = resolved_nic_modes(&bridge_then_macvtap, &cvm, 2); + assert_eq!(modes, vec![NetworkingMode::Bridge, NetworkingMode::Macvtap]); + + // A bridge TAP is netd's, and netd does not forward host ports. + let error = validate_port_mapping_nics(&[pinned(Some(0))], &modes).unwrap_err(); + assert!(error.to_string().contains("bridge"), "{error}"); + + let error = validate_port_mapping_nics(&[pinned(Some(1))], &modes).unwrap_err(); + assert!(error.to_string().contains("macvtap"), "{error}"); + assert!( + error.to_string().contains("cannot publish a host port"), + "{error}" + ); + + let error = validate_port_mapping_nics(&[pinned(Some(2))], &modes).unwrap_err(); + assert!(error.to_string().contains("this VM has 2"), "{error}"); + + // An unpinned mapping is never refused here: where it lands is + // resolved at launch, from the topology in force then. + validate_port_mapping_nics(&[pinned(None)], &modes).unwrap(); + + // An empty list is the node default's one NIC, resolved the same way a + // launch would resolve it rather than assumed to be user mode. + cvm.networking.nic.mode = NetworkingMode::Macvtap; + let modes = resolved_nic_modes(&[], &cvm, 2); + assert_eq!(modes, vec![NetworkingMode::Macvtap]); + assert!(validate_port_mapping_nics(&[pinned(Some(0))], &modes).is_err()); + } + #[test] fn create_without_networking_persists_following_default() { let manifest = @@ -882,74 +1490,1154 @@ mod tests { assert!(manifest.networks.is_empty()); } + #[test] + fn resize_request_rejects_empty_zero_and_empty_image_updates() { + let mut request = ResizeVmRequest { + id: "vm-1".into(), + ..Default::default() + }; + assert!(validate_resize_request(&request).is_err()); + + request.vcpu = Some(0); + assert!(validate_resize_request(&request).is_err()); + request.vcpu = Some(1); + assert!(validate_resize_request(&request).is_ok()); + + request.vcpu = None; + request.memory = Some(0); + assert!(validate_resize_request(&request).is_err()); + request.memory = None; + request.disk_size = Some(0); + assert!(validate_resize_request(&request).is_err()); + request.disk_size = None; + request.image = Some(String::new()); + assert!(validate_resize_request(&request).is_err()); + } + + #[test] + fn simulated_tee_is_selected_per_instance_and_implies_no_tee() { + let mut request = test_vm_configuration(); + request.simulated_tee = Some("dstack-amd-sev-snp".into()); + let mut config = test_cvm_config(); + config.tee_simulator = Some(dstack_types::TeeSimulatorConfig { + mock_attestation_seed: Some("11".repeat(32)), + ..Default::default() + }); + + let manifest = create_manifest_from_vm_config(request, &config).unwrap(); + + assert_eq!( + manifest.simulated_tee, + Some(dstack_types::TeeVariant::DstackAmdSevSnp) + ); + assert!(manifest.no_tee); + assert!(!manifest.swtpm); + } + + #[test] + fn swtpm_is_decided_at_deployment_from_key_provider_and_simulator() { + let cases = [ + (None, "tpm", true), + (Some("dstack-tdx"), "tpm", true), + (Some("dstack-gcp-tdx"), "tpm", false), + (Some("dstack-aws-nitro-tpm"), "tpm", false), + (Some("dstack-tdx"), "kms", false), + ]; + let mut config = test_cvm_config(); + config.tee_simulator = Some(dstack_types::TeeSimulatorConfig { + mock_attestation_seed: Some("11".repeat(32)), + ..Default::default() + }); + + for (simulated_tee, key_provider, expected) in cases { + let mut request = test_vm_configuration(); + request.simulated_tee = simulated_tee.map(str::to_string); + request.compose_file = serde_json::json!({ "key_provider": key_provider }).to_string(); + + let manifest = create_manifest_from_vm_config(request, &config).unwrap(); + + assert_eq!(manifest.swtpm, expected); + } + } + + #[test] + fn simulated_tee_requires_node_credentials() { + let mut request = test_vm_configuration(); + request.simulated_tee = Some("dstack-tdx".into()); + + let err = create_manifest_from_vm_config(request, &test_cvm_config()).unwrap_err(); + + assert!(err + .to_string() + .contains("tee simulator credentials are not configured")); + } + + #[test] + fn invalid_simulated_tee_is_rejected() { + let mut request = test_vm_configuration(); + request.simulated_tee = Some("not-a-platform".into()); + + let err = create_manifest_from_vm_config(request, &test_cvm_config()).unwrap_err(); + + assert!(err.to_string().contains("unsupported TEE variant")); + } + + #[test] + fn volume_extraction_keeps_other_compose_fields_opaque() -> Result<()> { + assert!(extract_verity_volumes("not json")?.is_empty()); + assert!(extract_verity_volumes(r#"{"future_manifest":true}"#)?.is_empty()); + + let compose = serde_json::json!({ + "unknown_future_field": { "anything": true }, + "verity_volumes": [{ + "source": "volume.img", + "verity_root": "11".repeat(32), + "target": "/run/volume" + }] + }); + let volumes = extract_verity_volumes(&compose.to_string())?; + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].verity_root, [0x11; 32]); + Ok(()) + } + #[test] fn explicit_user_networking_is_resolved_before_persist() { let mut request = test_vm_configuration(); request.networks = vec![rpc::NetworkingConfig { mode: "user".to_string(), bridge_name: String::new(), + ..Default::default() }]; let manifest = create_manifest_from_vm_config(request, &test_cvm_config()).unwrap(); assert_eq!(manifest.networks.len(), 1); assert_eq!(manifest.networks[0].mode, NetworkingMode::User); - assert!(!manifest.networks[0].net.is_empty()); + // The node's user-mode network parameters are not the VM's to store; + // the type it stores cannot carry them at all now. + assert!(manifest.networks[0].bridge.is_empty()); } #[test] - fn bridge_name_is_rejected_for_user_mode() { - let err = networks_from_proto(&[rpc::NetworkingConfig { - mode: "user".to_string(), - bridge_name: "dstack-br0".to_string(), - }]) + fn authorized_macvtap_preserves_parent_and_inherits_forwarding_mode() { + let mut cvm_config = test_cvm_config(); + cvm_config + .allowed_network_modes + .push(NetworkingMode::Macvtap); + cvm_config.allowed_macvtap_parents.push("eth0".to_string()); + cvm_config.networking.nic.parent = "node-default".to_string(); + cvm_config.networking.macvtap_mode = "private".to_string(); + let networks = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "macvtap".to_string(), + parent: "eth0".to_string(), + ..Default::default() + }], + &cvm_config, + ) + .unwrap(); + + assert_eq!(networks[0].mode, NetworkingMode::Macvtap); + assert_eq!(networks[0].parent, "eth0"); + + // The manifest keeps the parent, which is identity. The forwarding mode + // the node owns and supplies at every launch is not a field this type + // has. + let stored = resolve_requested_networks(&networks, &cvm_config, 4).unwrap(); + assert_eq!(stored[0].parent, "eth0"); + + let at_launch = resolve_networking(&stored[0], &cvm_config, 4); + assert_eq!(at_launch.nic.parent, "eth0"); + assert_eq!(at_launch.macvtap_mode, "private"); + + // Repointing the node's forwarding mode reaches the VM. + cvm_config.networking.macvtap_mode = "bridge".to_string(); + assert_eq!( + resolve_networking(&stored[0], &cvm_config, 4).macvtap_mode, + "bridge" + ); + } + + /// Node shapes a deployment can be reported against, each with the node + /// default fields that mode populates. + fn node_shapes() -> Vec<(&'static str, CvmConfig)> { + let mut bridge = test_cvm_config(); + bridge.networking.nic.mode = NetworkingMode::Bridge; + bridge.networking.nic.bridge = "br-node".into(); + + let mut macvtap = test_cvm_config(); + macvtap.networking.nic.mode = NetworkingMode::Macvtap; + macvtap.networking.nic.parent = "eth-node".into(); + macvtap.networking.macvtap_mode = "private".into(); + macvtap.allowed_network_modes.push(NetworkingMode::Macvtap); + + let user = test_cvm_config(); + vec![ + ("bridge node", bridge), + ("macvtap node", macvtap), + ("user node", user), + ] + } + + /// Every override a caller can express, including the tuning-only shape + /// that names no backend. + fn request_shapes(node: &CvmConfig) -> Vec<(String, rpc::NetworkingConfig)> { + let mode = networking_mode_name_for_test(node.networking.nic.mode); + // The user-mode backend has neither multiqueue nor vhost, and naming it + // and then asking for either is a refusal rather than a round-trip + // failure. Asking to turn vhost off is always legal. + let multiqueue = Networking::mode_supports_multiqueue(node.networking.nic.mode); + let vhost_on = Networking::mode_supports_vhost(node.networking.nic.mode).then_some(true); + // Naming a backend's identity field is only legal alongside a mode, so + // it varies with the named case. The node's own values are used because + // that is what GetInfo reports back. + let identity: &[(&str, &str, &str)] = &[ + ("", "", ""), + ("+ bridge", node.networking.nic.bridge.as_str(), ""), + ("+ parent", "", node.networking.nic.parent.as_str()), + ]; + let mut shapes = vec![]; + for (named, mode) in [("named", mode.to_string()), ("inherited", String::new())] { + for (tuning, vhost, queues) in [ + ("untuned", None, None), + ("vhost off", Some(false), None), + ("queues", None, multiqueue.then_some(2)), + ("both", vhost_on, multiqueue.then_some(2)), + ] { + for (label, bridge, parent) in identity { + // An inherited entry may not name a backend at all, which + // is a refusal covered by its own test. + let inherited = mode.is_empty(); + if inherited && !(bridge.is_empty() && parent.is_empty()) { + continue; + } + // Only the mode that owns a field may carry it. + let bridge_ok = node.networking.nic.mode == NetworkingMode::Bridge; + let parent_ok = node.networking.nic.mode == NetworkingMode::Macvtap; + if (!bridge.is_empty() && !bridge_ok) || (!parent.is_empty() && !parent_ok) { + continue; + } + shapes.push(( + format!("{named} + {tuning} {label}"), + rpc::NetworkingConfig { + mode: mode.clone(), + bridge_name: bridge.to_string(), + parent: parent.to_string(), + vhost, + queues, + ..Default::default() + }, + )); + } + } + } + shapes + } + + /// A backend's identity field without a mode would freeze whichever mode + /// the node had at deploy time, and then be reported alongside an empty + /// mode -- a combination the RPC itself rejects, which would leave the VM's + /// tuning permanently uneditable. + #[test] + fn an_inherited_entry_may_not_name_a_backend() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Macvtap; + cvm.networking.nic.parent = "eth-node".into(); + cvm.allowed_macvtap_parents.push("eth1".into()); + cvm.allowed_network_modes.push(NetworkingMode::Macvtap); + + let err = networking_from_proto( + &rpc::NetworkingConfig { + parent: "eth1".into(), + queues: Some(2), + ..Default::default() + }, + &cvm, + ) .unwrap_err(); + assert!(err.to_string().contains("networking mode is required")); - assert!(err.to_string().contains("bridge_name is only valid")); + // Naming the mode alongside it is fine. + networking_from_proto( + &rpc::NetworkingConfig { + mode: "macvtap".into(), + parent: "eth1".into(), + queues: Some(2), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("a named backend is an override"); } + fn networking_mode_name_for_test(mode: NetworkingMode) -> &'static str { + match mode { + NetworkingMode::Bridge => "bridge", + NetworkingMode::User => "user", + NetworkingMode::Macvtap => "macvtap", + NetworkingMode::Custom => "custom", + } + } + + /// `GetInfo` reports the configuration that `UpdateVm` and `UpgradeApp` + /// take back, and both vmm-cli and the web UI read it, change one field, + /// and resend the rest. So everything reportable has to be acceptable, and + /// accepting it has to land on the same VM. + /// + /// This asserts the property over every mode and tuning combination on + /// purpose. Asserting one shape is how an inherited `parent` on a bridge + /// NIC, and then `macvtap_mode` and `bridge_name`, each reached a release: + /// every one of them was a case the fixed example did not cover. #[test] - fn repeated_networks_rejects_empty_entries() { - let err = networks_from_proto(&[rpc::NetworkingConfig { - mode: String::new(), - bridge_name: String::new(), - }]) + fn everything_get_info_reports_is_accepted_back_unchanged() { + for (node_label, cvm) in node_shapes() { + for (shape_label, request) in request_shapes(&cvm) { + let case = format!("{node_label} / {shape_label}"); + let Some(requested) = networking_from_proto(&request, &cvm) + .unwrap_or_else(|error| panic!("{case}: deployment rejected: {error:#}")) + else { + // No override at all; nothing is recorded, nothing to report. + continue; + }; + // Skip the host-dependent bridge existence check: this is + // about what the RPC reports versus what it accepts. + let merged = resolve_networking(&requested, &cvm, 4); + let stored = manifest_networks(vec![merged.clone()], &[requested]); + + let reported = crate::app::networking_to_proto(&stored[0]); + let accepted = networking_from_proto(&reported, &cvm) + .unwrap_or_else(|error| { + panic!("{case}: GetInfo output was rejected on the way back: {error:#}") + }) + .unwrap_or_else(|| panic!("{case}: the override was lost in the round trip")); + + let restored = + manifest_networks(vec![resolve_networking(&accepted, &cvm, 4)], &[accepted]); + assert_eq!(stored, restored, "{case}: round trip changed the manifest"); + assert_eq!( + resolve_networking(&restored[0], &cvm, 4), + merged, + "{case}: round trip changed what the launch sees" + ); + } + } + } + + /// A VM must not become uneditable because its node moved somewhere its + /// tuning does not apply. The report and the deployment RPC have to agree + /// on every backend the node can be pointed at, not just the one it had + /// when the VM was deployed. + #[test] + fn an_inherited_override_still_round_trips_after_the_node_moves() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.networking.nic.bridge = "br-node".into(); + + let requested = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(4), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("tuning must produce an override"); + let stored = manifest_networks(vec![resolve_networking(&requested, &cvm, 8)], &[requested]); + + for mode in [ + NetworkingMode::User, + NetworkingMode::Custom, + NetworkingMode::Macvtap, + NetworkingMode::Bridge, + ] { + cvm.networking.nic.mode = mode; + cvm.networking.nic.parent = "eth-node".into(); + cvm.networking.netdev = "tap,id=net0,ifname=custom0".into(); + + let reported = crate::app::networking_to_proto(&stored[0]); + let accepted = networking_from_proto(&reported, &cvm) + .unwrap_or_else(|error| panic!("{mode:?}: report was rejected: {error:#}")) + .unwrap_or_else(|| panic!("{mode:?}: the override was lost")); + let restored = + manifest_networks(vec![resolve_networking(&accepted, &cvm, 8)], &[accepted]); + // An inherited entry still has to hold *some* mode -- every + // consumer matches on one -- and it is rewritten to whatever the + // node has now. Resolution ignores it, so compare what the launch + // sees rather than the field nothing reads. + assert!(restored[0].inherit_mode, "{mode:?}: pinned a backend"); + assert_eq!( + resolve_networking(&restored[0], &cvm, 8), + resolve_networking(&stored[0], &cvm, 8), + "{mode:?}: round trip changed what the launch sees" + ); + assert_eq!(restored[0].queues, Some(4), "{mode:?}: lost the request"); + } + } + + /// The counterpart: a node that changes its mind still reaches VMs that + /// never named a backend, and never reaches ones that did. + #[test] + fn a_node_backend_change_reaches_exactly_the_vms_that_inherited_it() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.networking.nic.bridge = "br-node".into(); + + let tuning_only = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(2), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("tuning must produce an override"); + let named = networking_from_proto( + &rpc::NetworkingConfig { + mode: "bridge".into(), + queues: Some(2), + ..Default::default() + }, + &cvm, + ) + .unwrap() + .expect("a named backend is an override"); + + let requests = vec![tuning_only, named]; + let merged = requests + .iter() + .map(|request| resolve_networking(request, &cvm, 4)) + .collect::>(); + let stored = manifest_networks(merged, &requests); + + // The operator repoints the node at a different backend. + cvm.networking.nic.mode = NetworkingMode::User; + assert_eq!( + resolve_networking(&stored[0], &cvm, 4).nic.mode, + NetworkingMode::User, + "a VM that never named a backend must follow the node" + ); + assert_eq!( + resolve_networking(&stored[1], &cvm, 4).nic.mode, + NetworkingMode::Bridge, + "a VM that named its backend keeps it for life" + ); + // User mode has no multiqueue backend, so the inherited NIC drops to + // one queue pair while it is there -- but the request is not lost. + assert_eq!(resolve_networking(&stored[0], &cvm, 4).queue_pairs(), 1); + assert_eq!(stored[0].queues, Some(2)); + cvm.networking.nic.mode = NetworkingMode::Bridge; + assert_eq!( + resolve_networking(&stored[0], &cvm, 4).queue_pairs(), + 2, + "tuning must survive an excursion through a backend that ignores it" + ); + } + + #[test] + fn queue_requests_are_bounded_by_node_policy() { + let mut cvm_config = test_cvm_config(); + cvm_config.max_net_queues = 4; + cvm_config.allowed_bridges.push("tenant-br0".to_string()); + let request = |queues: u32| { + [rpc::NetworkingConfig { + mode: "bridge".to_string(), + bridge_name: "tenant-br0".to_string(), + queues: Some(queues), + ..Default::default() + }] + }; + + let networks = networks_from_proto(&request(4), &cvm_config).unwrap(); + assert_eq!(networks[0].queues, Some(4)); + + let err = networks_from_proto(&request(5), &cvm_config).unwrap_err(); + assert!(err.to_string().contains("must not exceed 4")); + } + + /// A mode the deploy dialog offers has to be one the deployment RPC will + /// take. Offering one node policy forbids puts a choice in front of an + /// operator whose only outcome is "not allowed by node policy". + #[test] + fn advertised_modes_are_ones_the_rpc_would_accept() { + let mut cvm = test_cvm_config(); + cvm.allowed_network_modes = vec![NetworkingMode::User]; + for mode in ["bridge", "macvtap"] { + assert!( + networking_from_proto( + &rpc::NetworkingConfig { + mode: mode.to_string(), + ..Default::default() + }, + &cvm, + ) + .is_err(), + "{mode} should be refused by this policy" + ); + } + assert_eq!(advertised_modes(&cvm, true), vec!["user".to_string()]); + + cvm.allowed_network_modes = vec![NetworkingMode::User, NetworkingMode::Macvtap]; + assert_eq!( + advertised_modes(&cvm, true), + vec!["user".to_string(), "macvtap".to_string()] + ); + + // A mode policy allows but the host cannot serve is still not offered. + cvm.allowed_network_modes.push(NetworkingMode::Bridge); + assert!(!advertised_modes(&cvm, false).contains(&"bridge".to_string())); + assert!(advertised_modes(&cvm, true).contains(&"bridge".to_string())); + } + + /// A VM keeps its bridge for life, so the node dropping that bridge from + /// its own configuration must not make the VM's reported configuration + /// unsendable -- there is no flag anywhere to clear a field the caller + /// never typed. + /// Deployment allowlists govern what a caller may newly select. Widening + /// them from a VM's own holdings is what keeps read-modify-write working + /// across a node change -- but only from a field the entry's own mode + /// owns. Resolution used to copy the node's whole networking value into + /// every VM, so a bridge NIC carried whatever macvtap parent the node had + /// configured, and widening from that let the VM move to a parent policy + /// forbids. The split types make the node's other fields unreachable; the + /// two identity fields still share one type, so the scoping stays explicit. + #[test] + fn holdings_widen_policy_only_for_the_mode_that_owns_them() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.networking.nic.parent = "eth1".into(); + cvm.allowed_network_modes = vec![ + NetworkingMode::Bridge, + NetworkingMode::Macvtap, + NetworkingMode::User, + ]; + assert!(cvm.allowed_macvtap_parents.is_empty()); + + // What a bridge NIC deployed on this node used to end up holding. + let held = [NicNetworking { + mode: NetworkingMode::Bridge, + bridge: "br0".into(), + parent: "eth0".into(), + ..NicNetworking::default() + }]; + let widened = held_networking_config(&cvm, &held); + assert!(widened.allowed_bridges.contains(&"br0".to_string())); + assert!(!widened + .allowed_macvtap_parents + .contains(&"eth0".to_string())); + + let request = [rpc::NetworkingConfig { + mode: "macvtap".into(), + parent: "eth0".into(), + ..Default::default() + }]; + let err = networks_from_proto(&request, &widened).unwrap_err(); + assert!( + err.to_string().contains("not allowed by node policy"), + "{err}" + ); + } + + /// Lowering the node's queue cap bounds what a deployment may newly ask + /// for. It does not retune VMs already pinned above it, and those keep + /// reporting their count on every GetInfo -- so without this, lowering the + /// cap makes every networking update on such a VM fail on a field the + /// operator never typed, including the one that would unpin it. + #[test] + fn a_vm_may_restate_a_queue_count_the_node_has_since_capped() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.max_net_queues = 2; + + let held = [NicNetworking { + mode: NetworkingMode::Bridge, + queues: Some(8), + ..NicNetworking::default() + }]; + let request = [rpc::NetworkingConfig { + mode: "bridge".into(), + queues: Some(8), + vhost: Some(false), + ..Default::default() + }]; + + let err = networks_from_proto(&request, &cvm).unwrap_err(); + assert!(err.to_string().contains("must not exceed 2"), "{err}"); + + let widened = held_networking_config(&cvm, &held); + let networks = networks_from_proto(&request, &widened).unwrap(); + assert_eq!(networks[0].queues, Some(8)); + + // Only up to what it holds, though. + let more = [rpc::NetworkingConfig { + mode: "bridge".into(), + queues: Some(9), + ..Default::default() + }]; + let err = networks_from_proto(&more, &widened).unwrap_err(); + assert!(err.to_string().contains("must not exceed 8"), "{err}"); + } + + /// `optional uint32` tells an absent field from a typed zero, so reading + /// zero as "unset" would answer a request for no queues with the + /// vCPU-scaled default. Resize says the same about a zero vCPU count. + #[test] + fn an_explicit_zero_queue_count_is_an_error_not_an_absent_field() { + let cvm = test_cvm_config(); + let request = rpc::NetworkingConfig { + mode: "bridge".into(), + queues: Some(0), + ..Default::default() + }; + let err = networking_from_proto(&request, &cvm).unwrap_err(); + assert!(err.to_string().contains("must be at least 1"), "{err}"); + + // Including when it is the only thing the request says. That arm + // returns "no override at all" before anything reads the count, so a + // zero here used to be answered with up to sixteen queue pairs. + let bare = rpc::NetworkingConfig { + queues: Some(0), + ..Default::default() + }; + let err = networking_from_proto(&bare, &cvm).unwrap_err(); + assert!(err.to_string().contains("must be at least 1"), "{err}"); + } + + /// Resolution fills both identity fields from the node, so a bridge NIC's + /// resolved value carries whatever macvtap parent the node happens to have + /// configured. Storing that made every later reader responsible for knowing + /// which field its mode owns, and the one that widens deployment policy + /// from a VM's holdings got it wrong: repoint the node's parent and a + /// bridge VM could move itself to a parent policy forbids. + #[test] + fn a_vm_records_only_the_identity_field_its_own_mode_owns() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.networking.nic.bridge = "br-node".into(); + cvm.networking.nic.parent = "eth-node".into(); + + let networks = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "bridge".into(), + ..Default::default() + }], + &cvm, + ) + .unwrap(); + // `manifest_networks` directly: what a VM records is the question, and + // `resolve_requested_networks` would first validate the merged view + // against this host's real interfaces. + let merged = networks + .iter() + .map(|request| resolve_networking(request, &cvm, 4)) + .collect::>(); + let stored = manifest_networks(merged, &networks); + assert_eq!(stored[0].bridge, "br-node"); + assert!(stored[0].parent.is_empty(), "{:?}", stored[0]); + + // And the launch still gets the node's parent, because that half was + // never the VM's to hold in the first place. + let at_launch = resolve_networking(&stored[0], &cvm, 4); + assert_eq!(at_launch.nic.parent, "eth-node"); + } + + #[test] + fn a_vm_may_restate_a_bridge_it_already_holds() { + let mut cvm = test_cvm_config(); + cvm.networking.nic.mode = NetworkingMode::Bridge; + cvm.networking.nic.bridge = "br-new".into(); + assert!(cvm.allowed_bridges.is_empty()); + + let held = [NicNetworking { + mode: NetworkingMode::Bridge, + bridge: "br-old".into(), + ..NicNetworking::default() + }]; + let request = [rpc::NetworkingConfig { + mode: "bridge".into(), + bridge_name: "br-old".into(), + queues: Some(2), + ..Default::default() + }]; + + // Without the VM's own holdings this is a bridge it may not select. + let err = networks_from_proto(&request, &cvm).unwrap_err(); + assert!(err.to_string().contains("not allowed by node policy")); + + let widened = held_networking_config(&cvm, &held); + let networks = networks_from_proto(&request, &widened).unwrap(); + assert_eq!(networks[0].bridge, "br-old"); + + // And it is still only this VM's own values that are permitted. + let other = [rpc::NetworkingConfig { + mode: "bridge".into(), + bridge_name: "br-someone-else".into(), + ..Default::default() + }]; + assert!(networks_from_proto(&other, &widened).is_err()); + } + + /// vhost follows the same rule as the queue count: refused for a backend + /// the caller chose and that has none, accepted and dormant for one they + /// inherited. Accepting it silently on a chosen backend would leave the + /// deploy dialog reporting `vhost: on` next to a NIC running without it. + #[test] + fn vhost_is_refused_only_for_a_backend_the_caller_chose() { + let cvm_config = test_cvm_config(); + let err = networking_from_proto( + &rpc::NetworkingConfig { + mode: "user".into(), + vhost: Some(true), + ..Default::default() + }, + &cvm_config, + ) .unwrap_err(); + assert!(err.to_string().contains("no vhost data plane"), "{err:#}"); + + // Turning it off is a no-op that matches reality, so it is allowed. + networking_from_proto( + &rpc::NetworkingConfig { + mode: "user".into(), + vhost: Some(false), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); - assert!(err.to_string().contains("networking mode is required")); + // Inherited from a user-mode node: accepted, dormant, and live again + // when the node moves to a backend that has one. + let mut cvm_config = test_cvm_config(); + assert_eq!(cvm_config.networking.nic.mode, NetworkingMode::User); + let requested = networking_from_proto( + &rpc::NetworkingConfig { + vhost: Some(true), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert!(!resolve_networking(&requested, &cvm_config, 4).vhost_enabled()); + + cvm_config.networking.nic.mode = NetworkingMode::Bridge; + cvm_config.networking.nic.bridge = "br-node".into(); + assert!(resolve_networking(&requested, &cvm_config, 4).vhost_enabled()); } + /// A queue count is refused for a backend the caller chose and that cannot + /// honour it, because the caller can fix the request. It is accepted for + /// one they inherited, because they cannot: the node picked that backend + /// and may pick another tomorrow, and refusing would leave GetInfo + /// reporting a count nothing is allowed to send back. #[test] - fn repeated_networks_rejects_custom_entries() { - let err = networks_from_proto(&[rpc::NetworkingConfig { - mode: "custom".to_string(), - bridge_name: String::new(), - }]) + fn a_queue_count_is_refused_only_for_a_backend_the_caller_chose() { + let cvm_config = test_cvm_config(); + let err = networking_from_proto( + &rpc::NetworkingConfig { + mode: "user".into(), + queues: Some(4), + ..Default::default() + }, + &cvm_config, + ) .unwrap_err(); + assert!(err.to_string().contains("does not support multiple queues")); + + // Inherited: accepted, and dormant until the node moves to a backend + // that can honour it. + for mode in [NetworkingMode::Custom, NetworkingMode::User] { + let mut cvm_config = test_cvm_config(); + cvm_config.networking.nic.mode = mode; + cvm_config.networking.netdev = "tap,id=net0,ifname=custom0".into(); + let requested = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(4), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert_eq!(requested.queues, Some(4)); + assert_eq!( + resolve_networking(&requested, &cvm_config, 8).queue_pairs(), + 1, + "{mode:?} cannot carry multiqueue, whatever was asked for" + ); + + // And the request is still there when the node moves back. + cvm_config.networking.nic.mode = NetworkingMode::Bridge; + cvm_config.networking.nic.bridge = "br-node".into(); + assert_eq!( + resolve_networking(&requested, &cvm_config, 8).queue_pairs(), + 4 + ); + } + } - assert!(err.to_string().contains("custom networking mode")); + #[test] + fn user_mode_rejects_multiqueue_but_a_single_queue_is_fine() { + let cvm_config = test_cvm_config(); + let request = |queues: u32| { + [rpc::NetworkingConfig { + mode: "user".to_string(), + queues: Some(queues), + ..Default::default() + }] + }; + + networks_from_proto(&request(1), &cvm_config).unwrap(); + let err = networks_from_proto(&request(2), &cvm_config).unwrap_err(); + assert!(err.to_string().contains("does not support multiple queues")); } #[test] - fn multiple_bridges_are_rejected_when_builtin_forwarding_is_enabled() { + fn tuning_alone_keeps_the_node_backend_without_tripping_mode_policy() { let mut cvm_config = test_cvm_config(); - cvm_config.networking.forward_service_enabled = true; - let mut request = test_vm_configuration(); - request.networks = vec![ - rpc::NetworkingConfig { - mode: "bridge".to_string(), - bridge_name: "lo".to_string(), + // A backend the node uses but does not let callers choose. + cvm_config.networking.nic.mode = NetworkingMode::Macvtap; + cvm_config.networking.nic.parent = "eth0".to_string(); + assert!(!cvm_config + .allowed_network_modes + .contains(&NetworkingMode::Macvtap)); + + let networking = networking_from_proto( + &rpc::NetworkingConfig { + vhost: Some(false), + ..Default::default() }, - rpc::NetworkingConfig { - mode: "bridge".to_string(), - bridge_name: "lo".to_string(), + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert_eq!(networking.mode, NetworkingMode::Macvtap); + assert_eq!(networking.vhost, Some(false)); + + // Naming that backend explicitly is still a choice, and still denied. + let err = networking_from_proto( + &rpc::NetworkingConfig { + mode: "macvtap".to_string(), + vhost: Some(false), + ..Default::default() }, - ]; + &cvm_config, + ) + .unwrap_err(); + assert!(err.to_string().contains("not allowed by node policy")); + + // An untouched request still means "no override at all". + assert!( + networking_from_proto(&rpc::NetworkingConfig::default(), &cvm_config) + .unwrap() + .is_none() + ); + } - let err = create_manifest_from_vm_config(request, &cvm_config).unwrap_err(); + #[test] + fn an_inherited_backend_is_never_pinned_into_the_manifest() { + // Tuning must not become a way to pin a backend the caller was never + // allowed to choose, nor to freeze one the node still owns. + let mut cvm_config = test_cvm_config(); + cvm_config.networking.nic.mode = NetworkingMode::Macvtap; + cvm_config.networking.nic.parent = "eth0".to_string(); + cvm_config.networking.macvtap_mode = "private".to_string(); + + let requested = networking_from_proto( + &rpc::NetworkingConfig { + queues: Some(2), + ..Default::default() + }, + &cvm_config, + ) + .unwrap() + .expect("tuning must produce an override"); + assert!(requested.inherit_mode); + + let persisted = resolve_requested_networks(&[requested], &cvm_config, 4).unwrap(); + assert_eq!(persisted[0].queues, Some(2)); + // Nothing the node owns was copied in. The forwarding mode, the MAC + // prefix and the user-mode network parameters are not fields this type + // has any more; the parent is, and a NIC that named no backend does not + // get to keep the node's. + assert!(persisted[0].parent.is_empty()); + assert!(persisted[0].bridge.is_empty()); + + // Repointing the node moves the VM with it. + cvm_config.networking.nic.parent = "eth1".to_string(); + let at_launch = resolve_networking(&persisted[0], &cvm_config, 4); + assert_eq!(at_launch.nic.parent, "eth1"); + assert_eq!(at_launch.queue_pairs(), 2); + } - assert!(err - .to_string() - .contains("built-in port forwarding supports only one bridge")); + #[test] + fn deployment_pins_identity_but_not_data_plane_tuning() { + let mut cvm_config = test_cvm_config(); + cvm_config.networking.nic.vhost = Some(true); + cvm_config.networking.nic.queues = Some(2); + let networks = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "user".to_string(), + vhost: Some(false), + ..Default::default() + }], + &cvm_config, + ) + .unwrap(); + + let resolved = resolve_requested_networks(&networks, &cvm_config, 4).unwrap(); + // The backend the caller chose is pinned. + assert_eq!(resolved[0].mode, NetworkingMode::User); + assert!(!resolved[0].inherit_mode); + // The explicit request is kept. + assert_eq!(resolved[0].vhost, Some(false)); + // What the caller never asked for stays unset, so the node still owns + // it: an operator disabling vhost node-wide must reach this VM too. + assert_eq!(resolved[0].queues, None); + } + + #[test] + fn a_node_wide_vhost_rollback_reaches_a_vm_deployed_with_an_override() { + // macvtap keeps this independent of which interfaces the test host has. + let mut cvm_config = test_cvm_config(); + cvm_config + .allowed_network_modes + .push(NetworkingMode::Macvtap); + cvm_config.allowed_macvtap_parents.push("eth0".to_string()); + cvm_config.networking.nic.parent = "eth0".to_string(); + let networks = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "macvtap".to_string(), + parent: "eth0".to_string(), + ..Default::default() + }], + &cvm_config, + ) + .unwrap(); + let persisted = resolve_requested_networks(&networks, &cvm_config, 4).unwrap(); + assert!(persisted[0].vhost.is_none()); + + cvm_config.networking.nic.vhost = Some(false); + let at_launch = resolve_networking(&persisted[0], &cvm_config, 4); + assert!(!at_launch.vhost_enabled()); + } + + #[test] + fn macvtap_is_denied_by_default_rpc_policy() { + let err = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "macvtap".to_string(), + ..Default::default() + }], + &test_cvm_config(), + ) + .unwrap_err(); + + assert!(err.to_string().contains("not allowed by node policy")); + } + + #[test] + fn bridge_override_requires_allowlist_entry() { + let request = [rpc::NetworkingConfig { + mode: "bridge".to_string(), + bridge_name: "tenant-br0".to_string(), + ..Default::default() + }]; + let mut cvm_config = test_cvm_config(); + + let err = networks_from_proto(&request, &cvm_config).unwrap_err(); + assert!(err.to_string().contains("bridge_name 'tenant-br0'")); + + cvm_config.allowed_bridges.push("tenant-br0".to_string()); + let networks = networks_from_proto(&request, &cvm_config).unwrap(); + assert_eq!(networks[0].bridge, "tenant-br0"); + } + + #[test] + fn macvtap_parent_requires_allowlist_entry() { + let request = [rpc::NetworkingConfig { + mode: "macvtap".to_string(), + parent: "eth1".to_string(), + ..Default::default() + }]; + let mut cvm_config = test_cvm_config(); + cvm_config + .allowed_network_modes + .push(NetworkingMode::Macvtap); + + let err = networks_from_proto(&request, &cvm_config).unwrap_err(); + assert!(err.to_string().contains("macvtap parent 'eth1'")); + + cvm_config.allowed_macvtap_parents.push("eth1".to_string()); + let networks = networks_from_proto(&request, &cvm_config).unwrap(); + assert_eq!(networks[0].parent, "eth1"); + } + + #[test] + fn deployment_rpc_cannot_select_macvtap_forwarding_mode() { + let mut cvm_config = test_cvm_config(); + cvm_config + .allowed_network_modes + .push(NetworkingMode::Macvtap); + let err = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "macvtap".to_string(), + macvtap_mode: "passthru".to_string(), + ..Default::default() + }], + &cvm_config, + ) + .unwrap_err(); + + assert!(err.to_string().contains("node-controlled")); + } + + #[test] + fn bridge_name_is_rejected_for_user_mode() { + let err = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "user".to_string(), + bridge_name: "dstack-br0".to_string(), + ..Default::default() + }], + &test_cvm_config(), + ) + .unwrap_err(); + + assert!(err.to_string().contains("bridge_name is only valid")); + } + + #[test] + /// An entry in a list that overrides nothing describes a NIC that follows + /// the node entirely -- which is a thing an operator can mean, and the + /// only way the web UI can leave a NIC's backend unpinned. Rejecting it + /// would make an inherited NIC uneditable the moment its tuning is cleared. + fn repeated_networks_accepts_an_entry_that_overrides_nothing() { + let mut cvm_config = test_cvm_config(); + cvm_config.networking.nic.mode = NetworkingMode::Bridge; + cvm_config.networking.nic.bridge = "br-node".into(); + + let networks = networks_from_proto( + &[rpc::NetworkingConfig { + mode: String::new(), + bridge_name: String::new(), + ..Default::default() + }], + &cvm_config, + ) + .unwrap(); + assert_eq!(networks.len(), 1); + assert!(networks[0].inherit_mode); + assert_eq!(networks[0].queues, None); + assert!(networks[0].bridge.is_empty()); + + // It follows the node, like a VM with no networks at all. + cvm_config.networking.nic.mode = NetworkingMode::User; + assert_eq!( + resolve_networking(&networks[0], &cvm_config, 4).nic.mode, + NetworkingMode::User + ); + } + + #[test] + fn repeated_networks_rejects_custom_entries() { + let err = networks_from_proto( + &[rpc::NetworkingConfig { + mode: "custom".to_string(), + bridge_name: String::new(), + ..Default::default() + }], + &test_cvm_config(), + ) + .unwrap_err(); + + assert!(err.to_string().contains("custom networking mode")); + } + + #[test] + fn resolve_volume_source_rejects_escape_symlink_and_qemu_metachars() -> Result<()> { + let tmp = tempfile::tempdir()?; + let volumes = tmp.path().join("volumes"); + fs::create_dir_all(&volumes)?; + fs::write(volumes.join("ok.img"), b"ok")?; + let base = fs::canonicalize(&volumes)?; + + let ok = resolve_volume_source(&base, "ok.img")?; + assert_eq!(ok, base.join("ok.img")); + + let err = resolve_volume_source(&base, "../ok.img").unwrap_err(); + assert!(format!("{err:#}").contains("must be a bare file name")); + + fs::write(tmp.path().join("outside.img"), b"outside")?; + std::os::unix::fs::symlink(tmp.path().join("outside.img"), volumes.join("link.img"))?; + let err = resolve_volume_source(&base, "link.img").unwrap_err(); + assert!(format!("{err:#}").contains("escapes volumes_dir")); + + fs::write(volumes.join("bad,readonly=off"), b"bad")?; + let err = resolve_volume_source(&base, "bad,readonly=off").unwrap_err(); + assert!(format!("{err:#}").contains("',' or '='")); + + Ok(()) + } + + #[test] + fn resolve_volumes_resolves_measured_source() -> Result<()> { + let tmp = tempfile::tempdir()?; + fs::write(tmp.path().join("volume.img"), b"volume")?; + let mut cvm_config = test_cvm_config(); + cvm_config.volumes_dir = tmp.path().to_string_lossy().into_owned(); + + let volumes = resolve_volumes( + &[dstack_types::VerityVolume { + source: "volume.img".into(), + verity_root: [0; 32], + target: "/run/volume".into(), + }], + &cvm_config, + )?; + + assert_eq!(volumes.len(), 1); + assert_eq!( + volumes[0].source, + tmp.path().join("volume.img").display().to_string() + ); + Ok(()) + } + + #[test] + fn resolve_volumes_attaches_duplicate_root_once() -> Result<()> { + let tmp = tempfile::tempdir()?; + fs::write(tmp.path().join("first.img"), b"volume")?; + let mut cvm_config = test_cvm_config(); + cvm_config.volumes_dir = tmp.path().to_string_lossy().into_owned(); + let root = [7; 32]; + + let volumes = resolve_volumes( + &[ + dstack_types::VerityVolume { + source: "first.img".into(), + verity_root: root, + target: "/run/first".into(), + }, + dstack_types::VerityVolume { + // This source deliberately does not exist: the first entry + // owns the single attachment for this content root. + source: "duplicate.img".into(), + verity_root: root, + target: "/run/second".into(), + }, + ], + &cvm_config, + )?; + + assert_eq!(volumes.len(), 1); + assert_eq!( + volumes[0].source, + tmp.path().join("first.img").display().to_string() + ); + Ok(()) } } diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs new file mode 100644 index 000000000..e13facc73 --- /dev/null +++ b/dstack/vmm/src/netd.rs @@ -0,0 +1,1074 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Small privileged broker for TAP creation and libvirt nwfilter bindings. +//! +//! netd serves the `Netd` pRPC service (see `netd-rpc/proto/netd_rpc.proto`) +//! over a Unix socket. Each operation is its own RPC method with its own +//! request and response messages. + +use std::{ + fs::{File, OpenOptions, Permissions}, + io::{self, Write as _}, + os::{ + fd::AsRawFd, + unix::{ + fs::{FileTypeExt, PermissionsExt}, + net::UnixStream as StdUnixStream, + }, + }, + path::{Path, PathBuf}, + process::{Command, Stdio}, + sync::Arc, + time::Duration, +}; + +use anyhow::{anyhow, bail, Context, Result}; +use http_client::{prpc::PrpcClient, ConnectionReuse}; +use listenfd::ListenFd; +use ra_rpc::{CallContext, RpcCall}; +use rocket::listener::{unix::UnixStream as RocketUnixStream, Endpoint, Listener}; +use sha2::{Digest, Sha256}; +use tokio::net::UnixListener; +use tracing::{info, warn}; +use uuid::Uuid; +use wait_timeout::ChildExt; + +use dstack_netd_rpc::netd_server::{NetdRpc, NetdServer}; +pub use dstack_netd_rpc::{ + netd_client::NetdClient, CheckInterfaceRequest, InterfaceIdentity, PrepareBridgeRequest, + PrepareMacvtapRequest, PreparedInterface, +}; + +use crate::config::{NetdConfig, NetworkFilterConfig}; + +/// Bounds a whole call from the VMM, including waiting for netd to finish the +/// operations queued ahead of it. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +const COMMAND_TIMEOUT: Duration = Duration::from_secs(30); +const IP_PATH: &str = "/usr/sbin/ip"; +const VIRSH_PATH: &str = "/usr/bin/virsh"; +const LOCK_PATH: &str = "/run/lock/dstack-netd.lock"; +/// Upper bound on TAP queue pairs netd will create. Mirrors the VMM's own cap +/// so a malformed request cannot ask the kernel for an unbounded device. +const MAX_QUEUES: u32 = 64; + +pub fn tap_name(identity: &InterfaceIdentity) -> String { + let input = format!( + "{}\0{}\0{}", + identity.instance_id, identity.vm_id, identity.nic_index + ); + let digest = Sha256::digest(input.as_bytes()); + format!("dt{}", hex::encode(&digest[..6])) +} + +pub fn instance_id(configured: &str, run_path: &Path) -> String { + if !configured.trim().is_empty() { + return configured.trim().to_string(); + } + let digest = Sha256::digest(run_path.as_os_str().as_encoded_bytes()); + format!("path-{}", hex::encode(&digest[..8])) +} + +/// A client for the netd listening at `socket`. +/// +/// Every call opens its own connection. netd may have been restarted since the +/// last one, and a pooled connection to the previous process would fail a +/// request that a fresh one would have delivered. +pub fn client(socket: &Path) -> NetdClient { + NetdClient::new( + PrpcClient::new_unix(socket.display().to_string(), "/prpc".into()) + .with_connection_reuse(ConnectionReuse::Fresh) + .with_request_timeout(REQUEST_TIMEOUT), + ) +} + +/// Whether this error means netd was never reached. +/// +/// Only the two errnos that mean "nothing is listening". A socket the VMM's +/// user cannot open (`EACCES`) or a VMM out of descriptors is a different +/// problem, and must not be reported as a missing netd. +pub fn is_unreachable(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + cause.downcast_ref::().is_some_and(|error| { + matches!( + error.kind(), + io::ErrorKind::NotFound | io::ErrorKind::ConnectionRefused + ) + }) + }) +} + +pub async fn serve(config: NetdConfig) -> Result<()> { + if !nix::unistd::Uid::effective().is_root() { + bail!("netd must run as root"); + } + config.validate()?; + require_executable(IP_PATH)?; + require_executable(VIRSH_PATH)?; + let listener = match activated_listener()? { + Some(listener) => listener, + None => bind_listener(&config)?, + }; + info!(address = ?listener.local_addr()?, "netd listening"); + // Access is authorized by the Unix socket's owner, group, and mode. Any + // process that can connect is trusted with every netd method. + // Rocket's defaults only: a root daemon should not pick up a `Rocket.toml` + // from its working directory or `ROCKET_*` variables from its environment. + let figment = rocket::figment::Figment::from(rocket::Config::default()) + .merge(("ident", false)) + .merge(("cli_colors", false)); + let ignite = rocket::custom(figment) + .manage(NetdState::new(config, LOCK_PATH.into())) + .mount("/prpc", ra_rpc::prpc_routes!(NetdState, NetdHandler)) + .ignite() + .await + .map_err(|error| anyhow!("failed to ignite netd: {error}"))?; + ignite + .launch_on(NetdListener(listener)) + .await + .map_err(|error| anyhow!("netd server failed: {error}"))?; + Ok(()) +} + +/// Adapts the socket netd bound, or was handed by systemd, to Rocket. +struct NetdListener(UnixListener); + +impl Listener for NetdListener { + type Accept = RocketUnixStream; + type Connection = RocketUnixStream; + + async fn accept(&self) -> io::Result { + Ok(self.0.accept().await?.0) + } + + async fn connect(&self, accept: Self::Accept) -> io::Result { + Ok(accept) + } + + fn endpoint(&self) -> io::Result { + self.0.local_addr()?.try_into() + } +} + +fn activated_listener() -> Result> { + let mut listenfd = ListenFd::from_env(); + if listenfd.len() == 0 { + return Ok(None); + } + if listenfd.len() != 1 { + bail!( + "netd requires exactly one systemd-activated socket, received {}", + listenfd.len() + ); + } + let listener = listenfd + .take_unix_listener(0) + .context("systemd fd 0 is not a Unix stream listener")? + .context("systemd did not provide fd 0")?; + listener.set_nonblocking(true)?; + info!("using systemd-activated socket"); + Ok(Some(UnixListener::from_std(listener)?)) +} + +fn bind_listener(config: &NetdConfig) -> Result { + let listener = { + let _lock = OperationLock::acquire(Path::new(LOCK_PATH))?; + prepare_socket_path(&config.socket)?; + UnixListener::bind(&config.socket) + .with_context(|| format!("failed to bind netd socket {}", config.socket.display()))? + }; + std::fs::set_permissions(&config.socket, Permissions::from_mode(config.socket_mode))?; + Ok(listener) +} + +pub struct NetdState { + config: Arc, + /// Runs operations one at a time, in the order they arrived. + /// + /// The VMM relies on that order: after a Prepare times out on its side it + /// sends a removal for the same interface, which must run after the + /// Prepare it undoes rather than before it. Tokio's mutex queues waiters + /// fairly. [`OperationLock`] still serializes against other netd + /// processes, but an flock gives no ordering between its waiters. + serial: Arc>, + lock_path: Arc, +} + +impl NetdState { + fn new(config: NetdConfig, lock_path: PathBuf) -> Self { + Self { + config: Arc::new(config), + serial: Arc::new(tokio::sync::Mutex::new(())), + lock_path: Arc::new(lock_path), + } + } +} + +pub struct NetdHandler { + config: Arc, + serial: Arc>, + lock_path: Arc, +} + +impl RpcCall for NetdHandler { + type PrpcService = NetdServer; + + fn construct(context: CallContext<'_, NetdState>) -> Result { + Ok(Self { + config: context.state.config.clone(), + serial: context.state.serial.clone(), + lock_path: context.state.lock_path.clone(), + }) + } +} + +impl NetdHandler { + /// Runs one host operation under both locks. + /// + /// The operations shell out to `ip` and `virsh` and wait on them, so they + /// run on the blocking pool. The queue slot moves into that task: if the + /// request future is dropped, the next operation still waits for this one + /// to finish instead of starting beside it. + async fn run(self, operation: F) -> Result + where + T: Send + 'static, + F: FnOnce(&NetdConfig) -> Result + Send + 'static, + { + let slot = self.serial.lock_owned().await; + let config = self.config; + let lock_path = self.lock_path; + tokio::task::spawn_blocking(move || { + let _slot = slot; + let _lock = OperationLock::acquire(&lock_path)?; + operation(&config) + }) + .await + .context("netd operation task failed")? + } +} + +impl NetdRpc for NetdHandler { + async fn prepare_bridge(self, request: PrepareBridgeRequest) -> Result { + self.run(move |config| { + prepare_bridge(&config.libvirt_uri, &request, config.filter_policy()) + }) + .await + } + + async fn prepare_macvtap(self, request: PrepareMacvtapRequest) -> Result { + self.run(move |config| { + prepare_macvtap(&config.libvirt_uri, &request, config.filter_policy()) + }) + .await + } + + async fn remove_interface(self, identity: InterfaceIdentity) -> Result<()> { + self.run(move |config| { + validate_identity(&identity)?; + let tap = tap_name(&identity); + // Best effort about the binding, whatever was built. The strict + // rule exists for prepare, where a binding left at the name would + // block the one about to be created; at removal nothing is about + // to take the name, and failing here would leave the interface + // itself up on the bridge rather than just a stale binding. + remove_interface(&config.libvirt_uri, &tap, BindingCleanup::BestEffort) + }) + .await + } + + async fn check_interface(self, request: CheckInterfaceRequest) -> Result<()> { + self.run(move |config| { + let identity = required_identity(&request.identity)?; + let tap = tap_name(identity); + if !Path::new("/sys/class/net").join(&tap).exists() { + bail!("TAP {tap} does not exist"); + } + // An unfiltered TAP has no binding to dump; asking for one would + // report a healthy multiqueue interface as broken. + if request.filtered && !is_macvtap(&tap) { + virsh( + &config.libvirt_uri, + &["nwfilter-binding-dumpxml", &tap], + None, + )?; + } + Ok(()) + }) + .await + } +} + +/// The identity a request names, validated. +/// +/// Message fields are optional on the wire, and an interface name derived from +/// a missing identity would name some interface nobody asked for. +fn required_identity(identity: &Option) -> Result<&InterfaceIdentity> { + let identity = identity + .as_ref() + .context("request does not name an interface identity")?; + validate_identity(identity)?; + Ok(identity) +} + +fn prepare_macvtap( + libvirt_uri: &str, + request: &PrepareMacvtapRequest, + filter: &NetworkFilterConfig, +) -> Result { + let identity = required_identity(&request.identity)?; + let parent = request.parent.as_str(); + let qemu_uid = request.qemu_uid; + validate_name("parent", parent, 15, "_.-")?; + if !Path::new("/sys/class/net").join(parent).exists() { + bail!("parent interface {parent} does not exist"); + } + // A macvtap parent may be a bridge, and libvirt nwfilter does not apply to + // macvtap. So on a node that requires every bridge TAP to be filtered, a + // macvtap request naming that same bridge is the identical unfiltered L2 + // access the policy exists to refuse, spelled with a different operation. + // An interface enslaved to a bridge reaches the same segment. + if filter.requires_binding() { + let sysfs = Path::new("/sys/class/net").join(parent); + if sysfs.join("bridge").exists() { + bail!("this netd requires filtering, so {parent} may not be a macvtap parent: it is a host bridge"); + } + if sysfs.join("master").exists() { + bail!("this netd requires filtering, so {parent} may not be a macvtap parent: it is enslaved to a bridge"); + } + } + validate_mac(&request.mac)?; + let mac = request.mac.as_str(); + let mode = if request.mode.is_empty() { + "private" + } else { + request.mode.as_str() + }; + if !matches!(mode, "private" | "bridge" | "vepa" | "passthru") { + bail!("invalid macvtap mode"); + } + let queues = validate_queues(request.queues)?; + let tap = tap_name(identity); + remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort)?; + let queue_count = queues.to_string(); + let mut add = vec!["link", "add", "link", parent, "name", &tap, "address", mac]; + if queues > 1 { + // macvtap defaults to a single hardware queue pair. Without this the + // extra tap queues exist but the lower device still serializes. + add.extend_from_slice(&["numtxqueues", &queue_count, "numrxqueues", &queue_count]); + } + add.extend_from_slice(&["type", "macvtap", "mode", mode]); + ip(&add)?; + let result = (|| { + let ifindex = + std::fs::read_to_string(Path::new("/sys/class/net").join(&tap).join("ifindex")) + .context("failed to read macvtap ifindex")?; + let device = format!("/dev/tap{}", ifindex.trim()); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !Path::new(&device).exists() { + if std::time::Instant::now() >= deadline { + bail!("timed out waiting for macvtap device {device}"); + } + std::thread::sleep(Duration::from_millis(25)); + } + let metadata = std::fs::symlink_metadata(&device) + .with_context(|| format!("failed to inspect macvtap device {device}"))?; + if !metadata.file_type().is_char_device() { + bail!("macvtap device {device} is not a character device"); + } + std::os::unix::fs::chown(&device, Some(qemu_uid), None) + .with_context(|| format!("failed to set owner of macvtap device {device}"))?; + ip(&["link", "set", "dev", &tap, "up"])?; + Ok(device) + })(); + match result { + Ok(device) => { + info!(%tap, %parent, %mode, %device, %queues, "prepared macvtap"); + Ok(PreparedInterface { + tap, + device, + queues, + }) + } + Err(error) => { + let _ = remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort); + Err(error) + } + } +} + +struct OperationLock(File); + +impl OperationLock { + fn acquire(path: &Path) -> Result { + let file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(path) + .with_context(|| format!("failed to open {}", path.display()))?; + // SAFETY: flock only acts on the valid file descriptor owned by file. + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) } != 0 { + return Err(std::io::Error::last_os_error()).context("failed to lock netd operations"); + } + Ok(Self(file)) + } +} + +impl Drop for OperationLock { + fn drop(&mut self) { + // SAFETY: the file remains alive until after Drop returns. + unsafe { + libc::flock(self.0.as_raw_fd(), libc::LOCK_UN); + } + } +} + +fn prepare_bridge( + libvirt_uri: &str, + request: &PrepareBridgeRequest, + filter: &NetworkFilterConfig, +) -> Result { + let identity = validate_prepare_bridge(request, filter)?; + let filtered = request.filtered; + let tap = tap_name(identity); + // A failed VMM start may leave a deterministic resource behind. Replacing + // it makes prepare idempotent without accepting a caller-selected TAP. + // A binding outlives the interface it was bound to and TAP names are + // derived, so the same name comes back: clear whatever is there. Insist + // only when this prepare is about to create a replacement libvirt would + // refuse as a duplicate. + remove_interface( + libvirt_uri, + &tap, + if filtered { + BindingCleanup::Required + } else { + BindingCleanup::BestEffort + }, + )?; + + let uid = request.qemu_uid.to_string(); + let queues = validate_queues(request.queues)?; + let mut add = vec!["tuntap", "add", "dev", &tap, "mode", "tap"]; + if queues > 1 { + // QEMU refuses to attach when the device's IFF_MULTI_QUEUE state does + // not match its own `queues=` argument, in either direction. + add.push("multi_queue"); + } + add.extend_from_slice(&["user", &uid]); + ip(&add)?; + let result = (|| { + ip(&["link", "set", "dev", &tap, "master", &request.bridge])?; + if filtered { + let xml = binding_xml(identity, &request.mac, &tap, filter); + virsh( + libvirt_uri, + &["nwfilter-binding-create", "--validate", "/dev/stdin"], + Some(xml.as_bytes()), + )?; + } + ip(&["link", "set", "dev", &tap, "up"])?; + Ok(()) + })(); + if let Err(error) = result { + let _ = remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort); + return Err(error); + } + info!(%tap, bridge = %request.bridge, %filtered, %queues, "prepared TAP"); + Ok(PreparedInterface { + tap, + device: String::new(), + queues, + }) +} + +/// How hard removal must try to clear an nwfilter binding. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BindingCleanup { + /// The binding must be gone before this returns, because the caller is + /// about to create one at the same interface name and libvirt refuses a + /// duplicate. + Required, + /// Delete a binding if libvirt can be reached, but do not fail the removal + /// when it cannot. Unfiltered TAPs live on nodes where `libvirtd` need not + /// be running at all, and a stale binding left by an earlier, filtered + /// interface at this name is still worth clearing when it is. + BestEffort, +} + +fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Result<()> { + let macvtap = is_macvtap(tap); + if Path::new("/sys/class/net").join(tap).exists() { + let _ = ip(&["link", "set", "dev", tap, "down"]); + } + // A macvtap interface never carries a binding. Anything else might: this + // name may have been a filtered bridge TAP before, and the binding + // outlives the interface. + if !macvtap { + match cleanup { + BindingCleanup::Required => delete_binding(libvirt_uri, tap)?, + BindingCleanup::BestEffort => { + // netd refuses to start without virsh, so the binary is always + // here; libvirtd need not be running, and on a node that only + // wants macvtap or multiqueue it usually is not. + if let Err(error) = delete_binding(libvirt_uri, tap) { + warn!(%tap, "could not clear a possible nwfilter binding: {error:#}"); + } + } + } + } + if Path::new("/sys/class/net").join(tap).exists() { + ip(&["link", "delete", "dev", tap])?; + info!(%tap, "removed managed network interface"); + } + Ok(()) +} + +fn is_macvtap(interface: &str) -> bool { + Path::new("/sys/class/net") + .join(interface) + .join("macvtap") + .exists() +} + +/// Deletes an interface's nwfilter binding, if it has one. +/// +/// Goes through the same `COMMAND_TIMEOUT`-bounded helper as every other virsh +/// call. netd runs one operation at a time, so an unbounded call here +/// would let one unreachable libvirt stall every other VM's prepare and remove. +fn delete_binding(uri: &str, tap: &str) -> Result<()> { + match virsh(uri, &["nwfilter-binding-delete", tap], None) { + Ok(()) => Ok(()), + // Removal is idempotent. Having no binding is the normal case for + // macvtap, for unfiltered multiqueue TAPs, and for any name being + // reused after an earlier removal already cleared it. + Err(error) + if error + .to_string() + .contains("Network filter binding not found") => + { + Ok(()) + } + Err(error) => Err(error).context(format!("virsh failed to delete binding {tap}")), + } +} + +fn binding_xml( + identity: &InterfaceIdentity, + mac: &str, + tap: &str, + filter: &NetworkFilterConfig, +) -> String { + let owner_uuid = stable_uuid(identity); + let owner_name = format!( + "dstack:{}:{}:{}", + identity.instance_id, identity.vm_id, identity.nic_index + ); + let mut parameters = String::new(); + for (name, value) in &filter.parameters { + parameters.push_str(&format!( + "", + xml_escape(name), + xml_escape(value) + )); + } + format!( + "{}{}\ + \ + {}", + xml_escape(&owner_name), + owner_uuid, + xml_escape(tap), + xml_escape(mac), + xml_escape(&filter.filter), + parameters + ) +} + +fn stable_uuid(identity: &InterfaceIdentity) -> Uuid { + let digest = Sha256::digest( + format!( + "{}\0{}\0{}", + identity.instance_id, identity.vm_id, identity.nic_index + ) + .as_bytes(), + ); + let mut bytes = [0_u8; 16]; + bytes.copy_from_slice(&digest[..16]); + bytes[6] = (bytes[6] & 0x0f) | 0x50; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + Uuid::from_bytes(bytes) +} + +/// Validates a bridge prepare and returns the identity it names. +fn validate_prepare_bridge<'a>( + request: &'a PrepareBridgeRequest, + filter: &NetworkFilterConfig, +) -> Result<&'a InterfaceIdentity> { + let identity = required_identity(&request.identity)?; + validate_name("bridge", &request.bridge, 15, "_.-")?; + // Unfiltered bridge TAPs exist for unfiltered multiqueue, and netd holds + // that policy itself rather than trusting the caller with it. netd is the + // privileged side of this socket; on a node configured to filter bridge + // traffic, "build me a TAP on br0 with no nwfilter binding" is precisely + // the request the boundary exists to refuse, and anything that can reach + // the socket can make it. + // + // Refused before the host is inspected: this is about the request, not + // about what happens to exist on this machine. + if filter.requires_binding() && !request.filtered { + bail!("this netd requires an nwfilter binding on every bridge TAP"); + } + if !Path::new("/sys/class/net") + .join(&request.bridge) + .join("bridge") + .exists() + { + bail!("{} is not a host bridge", request.bridge); + } + validate_mac(&request.mac)?; + Ok(identity) +} + +fn validate_identity(identity: &InterfaceIdentity) -> Result<()> { + for (label, value) in [ + ("instance ID", identity.instance_id.as_str()), + ("VM ID", identity.vm_id.as_str()), + ] { + if value.is_empty() || value.len() > 128 || value.contains('\0') { + bail!("invalid {label}"); + } + } + if identity.nic_index > 255 { + bail!("NIC index is out of range"); + } + Ok(()) +} + +/// Normalizes a requested queue pair count. Zero means the caller did not ask +/// for multiqueue, which is the same device shape as one queue pair. +fn validate_queues(queues: u32) -> Result { + if queues > MAX_QUEUES { + bail!("queues must not exceed {MAX_QUEUES}"); + } + Ok(queues.max(1)) +} + +fn validate_name(label: &str, value: &str, max: usize, punctuation: &str) -> Result<()> { + if value.is_empty() + || value.len() > max + // `.` and `..` pass the charset check below, and every name validated + // here is then joined onto a sysfs path to ask whether the interface + // exists. `/sys/class/net/..` exists, so the question would be answered + // about a directory rather than about an interface. + || value == "." + || value == ".." + || !value + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || punctuation.contains(ch)) + { + bail!("invalid {label}"); + } + Ok(()) +} + +fn validate_mac(mac: &str) -> Result<()> { + let bytes = mac + .split(':') + .map(|part| { + if part.len() != 2 { + bail!("invalid MAC address"); + } + u8::from_str_radix(part, 16).context("invalid MAC address") + }) + .collect::>>()?; + if bytes.len() != 6 || bytes[0] & 1 != 0 { + bail!("invalid unicast MAC address"); + } + Ok(()) +} + +fn xml_escape(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('\'', "'") + .replace('"', """) +} + +fn ip(args: &[&str]) -> Result<()> { + run_command(IP_PATH, args, None) +} + +fn virsh(uri: &str, args: &[&str], stdin: Option<&[u8]>) -> Result<()> { + let mut full_args = vec!["--connect", uri]; + full_args.extend_from_slice(args); + run_command(VIRSH_PATH, &full_args, stdin) +} + +fn run_command(program: &str, args: &[&str], stdin: Option<&[u8]>) -> Result<()> { + run_command_with_timeout(program, args, stdin, COMMAND_TIMEOUT) +} + +fn run_command_with_timeout( + program: &str, + args: &[&str], + stdin: Option<&[u8]>, + command_timeout: Duration, +) -> Result<()> { + let mut child = Command::new(program) + .args(args) + .stdin(if stdin.is_some() { + Stdio::piped() + } else { + Stdio::null() + }) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to execute {program}"))?; + if let Some(input) = stdin { + child + .stdin + .take() + .context("missing command stdin")? + .write_all(input)?; + } + if child.wait_timeout(command_timeout)?.is_none() { + let _ = child.kill(); + let _ = child.wait(); + bail!( + "{} timed out after {command_timeout:?}", + Path::new(program).display() + ); + } + let output = child.wait_with_output()?; + if !output.status.success() { + let error = String::from_utf8_lossy(&output.stderr); + bail!("{} failed: {}", Path::new(program).display(), error.trim()); + } + Ok(()) +} + +fn require_executable(path: &str) -> Result<()> { + if !Path::new(path).is_file() { + bail!("required executable {path} does not exist"); + } + Ok(()) +} + +fn prepare_socket_path(socket: &Path) -> Result<()> { + let parent = socket.parent().context("netd socket has no parent")?; + std::fs::create_dir_all(parent) + .with_context(|| format!("failed to create {}", parent.display()))?; + if socket.exists() { + if StdUnixStream::connect(socket).is_ok() { + bail!("another netd is listening at {}", socket.display()); + } + std::fs::remove_file(socket) + .with_context(|| format!("failed to remove stale socket {}", socket.display()))?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeMap; + + fn identity(instance: &str, vm: &str, nic_index: u32) -> InterfaceIdentity { + InterfaceIdentity { + instance_id: instance.into(), + vm_id: vm.into(), + nic_index, + } + } + + #[test] + fn tap_names_are_stable_bounded_and_namespaced() { + let first = tap_name(&identity("one", "vm", 0)); + assert_eq!(first, tap_name(&identity("one", "vm", 0))); + assert_ne!(first, tap_name(&identity("two", "vm", 0))); + assert_ne!(first, tap_name(&identity("one", "vm", 1))); + assert!(first.len() <= 15); + } + + #[test] + fn binding_xml_escapes_values() { + let request = PrepareBridgeRequest { + identity: Some(identity("instance<&", "vm", 0)), + bridge: "br0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filtered: true, + queues: 0, + }; + let filter = NetworkFilterConfig { + mode: crate::config::NetworkFilterMode::Libvirt, + filter: "clean-traffic".into(), + parameters: BTreeMap::from([("IP".into(), "10.0.0.2<&".into())]), + }; + let xml = binding_xml( + request.identity.as_ref().unwrap(), + &request.mac, + "dt123", + &filter, + ); + assert!(xml.contains("instance<&")); + assert!(xml.contains("10.0.0.2<&")); + assert!(!xml.contains("instance<&")); + } + + #[test] + fn validation_rejects_injected_host_names() { + assert!(validate_name("bridge", "br0;id", 15, "_.-").is_err()); + // `/sys/class/net/..` exists, so an existence check on this name would + // answer about a directory rather than about an interface. + assert!(validate_name("parent", "..", 15, "_.-").is_err()); + assert!(validate_name("parent", ".", 15, "_.-").is_err()); + assert!(validate_name("filter", "../../filter", 128, "_.:-").is_err()); + assert!(validate_mac("ff:ff:ff:ff:ff:ff").is_err()); + } + + #[test] + fn queue_counts_normalize_to_at_least_one_and_stay_bounded() { + assert_eq!(validate_queues(0).unwrap(), 1); + assert_eq!(validate_queues(1).unwrap(), 1); + assert_eq!(validate_queues(MAX_QUEUES).unwrap(), MAX_QUEUES); + assert!(validate_queues(MAX_QUEUES + 1).is_err()); + } + + /// netd is the privileged side of this socket. "Build me a TAP on br0 with + /// no nwfilter binding" is the request the boundary exists to refuse on a + /// filtering node, and before this the daemon simply did what it was told, + /// leaving the invariant with the unprivileged caller. + #[test] + fn a_filtering_node_refuses_an_unfiltered_bridge_tap() { + let request = PrepareBridgeRequest { + identity: Some(identity("i", "v", 0)), + // A name no host has, so the check after this one is the one that + // fails when this one does not. + bridge: "dstack-nobr0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filtered: false, + queues: 4, + }; + let filtering = NetworkFilterConfig { + mode: crate::config::NetworkFilterMode::Libvirt, + ..NetworkFilterConfig::default() + }; + let error = validate_prepare_bridge(&request, &filtering).unwrap_err(); + assert!( + error.to_string().contains("requires an nwfilter binding"), + "{error}" + ); + + // An unfiltered node still builds them; that is what multiqueue needs. + // It gets as far as asking the host about the bridge, which is the + // next check and not this one's business. + let error = validate_prepare_bridge(&request, &NetworkFilterConfig::default()).unwrap_err(); + assert!( + error.to_string().contains("is not a host bridge"), + "{error}" + ); + } + + /// nwfilter does not apply to macvtap, and a macvtap parent may be the very + /// bridge the policy protects. Refusing an unfiltered bridge TAP while + /// handing out a macvtap on the same segment would leave the policy + /// enforced only against the spelling that happens to be checked. + #[test] + fn a_filtering_node_refuses_a_macvtap_parent_that_is_a_bridge() { + let filtering = NetworkFilterConfig { + mode: crate::config::NetworkFilterMode::Libvirt, + ..NetworkFilterConfig::default() + }; + let bridges: Vec = std::fs::read_dir("/sys/class/net") + .into_iter() + .flatten() + .flatten() + .filter(|entry| entry.path().join("bridge").exists()) + .filter_map(|entry| entry.file_name().into_string().ok()) + .collect(); + let Some(bridge) = bridges.first() else { + // Nothing to assert against on a host with no bridge; the unit + // below still pins the enslaved case's sysfs predicate. + return; + }; + let request = PrepareMacvtapRequest { + identity: Some(identity("i", "v", 0)), + parent: bridge.clone(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + mode: "bridge".into(), + queues: 4, + }; + let error = match prepare_macvtap("test:///default", &request, &filtering) { + Err(error) => error, + Ok(_) => panic!("a filtering node must not build a macvtap on a host bridge"), + }; + assert!(error.to_string().contains("is a host bridge"), "{error}"); + } + + /// The request says whether to bind a filter, never which one. `allow-arp` + /// contains no drop rule at all, and `clean-traffic` pinned to the + /// gateway's MAC and IP through its parameters filters nothing useful + /// either -- both would satisfy a policy that only asked for "some filter". + #[test] + fn the_bound_filter_comes_from_netds_own_configuration() { + let request = PrepareBridgeRequest { + identity: Some(identity("i", "v", 0)), + bridge: "br0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filtered: true, + queues: 1, + }; + // Nothing on the wire can name a filter: the field does not exist. + let wire = serde_json::to_value(&request).unwrap(); + assert!(wire.get("filter").is_none(), "{wire}"); + assert!(wire.get("parameters").is_none(), "{wire}"); + + let policy = NetworkFilterConfig { + mode: crate::config::NetworkFilterMode::Libvirt, + filter: "clean-traffic".into(), + parameters: BTreeMap::from([("IP".into(), "10.0.0.2".into())]), + }; + let xml = binding_xml( + request.identity.as_ref().unwrap(), + &request.mac, + "dt123", + &policy, + ); + assert!(xml.contains("filter='clean-traffic'"), "{xml}"); + assert!(xml.contains("value='10.0.0.2'"), "{xml}"); + } + + #[test] + fn command_timeout_kills_stalled_tool() { + let started = std::time::Instant::now(); + let error = run_command_with_timeout( + "/bin/sh", + &["-c", "sleep 10"], + None, + Duration::from_millis(50), + ) + .unwrap_err(); + assert!(error.to_string().contains("timed out")); + assert!(started.elapsed() < Duration::from_secs(2)); + } + + /// The VMM tells "netd is not running" apart from "netd refused". Both + /// ways a Unix socket can have nobody behind it must read as the former, + /// through the context the pRPC client stacks on the transport error. + #[tokio::test] + async fn a_socket_with_no_netd_behind_it_is_unreachable() { + let dir = tempfile::tempdir().unwrap(); + + let missing = dir.path().join("missing.sock"); + let error = client(&missing) + .remove_interface(identity("i", "v", 0)) + .await + .unwrap_err(); + assert!(is_unreachable(&error), "{error:#}"); + + // A socket file left behind by a netd that exited. + let stale = dir.path().join("stale.sock"); + drop(std::os::unix::net::UnixListener::bind(&stale).unwrap()); + let error = client(&stale) + .remove_interface(identity("i", "v", 0)) + .await + .unwrap_err(); + assert!(is_unreachable(&error), "{error:#}"); + + let refused = anyhow::anyhow!("this netd requires an nwfilter binding on every bridge TAP"); + assert!(!is_unreachable(&refused)); + } + + /// After a Prepare times out on the VMM's side, the VMM removes the same + /// interface. That removal must run after the Prepare, not beside or + /// before it, or the Prepare would leave behind the interface the removal + /// was sent to delete. + #[tokio::test] + async fn operations_run_one_at_a_time_in_arrival_order() { + let dir = tempfile::tempdir().unwrap(); + let state = NetdState::new(NetdConfig::default(), dir.path().join("netd.lock")); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + let mut tasks = Vec::new(); + for index in 0..8 { + let handler = NetdHandler { + config: state.config.clone(), + serial: state.serial.clone(), + lock_path: state.lock_path.clone(), + }; + let order = order.clone(); + tasks.push(tokio::spawn(handler.run(move |_| { + order.lock().unwrap().push((index, "start")); + // Long enough that a later operation running beside this one + // would interleave with it. + std::thread::sleep(Duration::from_millis(if index == 0 { 100 } else { 5 })); + order.lock().unwrap().push((index, "end")); + Ok(()) + }))); + // Let this request take its place in the queue before the next + // one arrives. + tokio::time::sleep(Duration::from_millis(5)).await; + } + for task in tasks { + task.await.unwrap().unwrap(); + } + let expected: Vec<_> = (0..8).flat_map(|i| [(i, "start"), (i, "end")]).collect(); + assert_eq!(*order.lock().unwrap(), expected); + } + + /// Serves netd's routes on a temporary socket, without the root and host + /// tool checks `serve` makes. + async fn spawn_test_netd(dir: &Path) -> PathBuf { + let socket = dir.join("netd.sock"); + let listener = UnixListener::bind(&socket).unwrap(); + let ignite = rocket::custom(rocket::Config::figment().merge(("log_level", "off"))) + .manage(NetdState::new(NetdConfig::default(), dir.join("netd.lock"))) + .mount("/prpc", ra_rpc::prpc_routes!(NetdState, NetdHandler)) + .ignite() + .await + .unwrap(); + tokio::spawn(ignite.launch_on(NetdListener(listener))); + socket + } + + /// A refusal has to reach the VMM with netd's reason in it: that reason is + /// what an operator acts on, and a bare status code names nothing. + #[tokio::test] + async fn a_refused_request_reaches_the_caller_with_netds_reason() { + let dir = tempfile::tempdir().unwrap(); + let socket = spawn_test_netd(dir.path()).await; + + // Refused by validation, before any host tool runs. + let error = client(&socket) + .check_interface(CheckInterfaceRequest { + identity: None, + filtered: false, + }) + .await + .unwrap_err(); + assert!(!is_unreachable(&error), "{error:#}"); + assert!( + format!("{error:#}").contains("does not name an interface identity"), + "{error:#}" + ); + + let error = client(&socket) + .remove_interface(identity("", "v", 0)) + .await + .unwrap_err(); + assert!( + format!("{error:#}").contains("invalid instance ID"), + "{error:#}" + ); + } +} diff --git a/dstack/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs index 3773fbcb2..62c4bd78b 100644 --- a/dstack/vmm/src/one_shot.rs +++ b/dstack/vmm/src/one_shot.rs @@ -2,10 +2,14 @@ // // SPDX-License-Identifier: Apache-2.0 -use crate::app::{make_sys_config, Image, VmConfig, VmWorkDir}; +use crate::app::{ + make_sys_config, needs_netd_interface, resolved_networks, settle_vhost, + simulator_config_for_manifest, sync_tee_simulator_config, Image, VmConfig, VmWorkDir, +}; use crate::config::Config; use crate::main_service; use anyhow::{Context, Result}; +use fs_err as fs; pub async fn run_one_shot( vm_config_path: &str, @@ -238,21 +242,9 @@ Compose file content (first 200 chars): let app_compose = vm_work_dir .app_compose() .context("Failed to get app compose")?; - let platform = config.cvm.resolved_platform(); - let use_mr_config_v3 = !manifest.no_tee - && (platform == crate::config::TeePlatform::AmdSevSnp - || (platform == crate::config::TeePlatform::Tdx - && config.cvm.use_mrconfigid - && !app_compose.key_provider_id.is_empty())); - let mr_config = if use_mr_config_v3 { - Some( - vm_work_dir - .prepare_mr_config_v3(&app_compose) - .context("Failed to prepare mr_config")?, - ) - } else { - None - }; + let mr_config = vm_work_dir + .prepare_mr_config(&manifest, &config.cvm, &app_compose) + .context("Failed to prepare mr_config")?; let sys_config_str = make_sys_config( &config, &manifest, @@ -261,7 +253,13 @@ Compose file content (first 200 chars): app_compose.requirements.as_ref(), )?; let sys_config_path = vm_work_dir.shared_dir().join(".sys-config.json"); - fs_err::write(&sys_config_path, sys_config_str).context("Failed to write sys config")?; + fs_err::write(&sys_config_path, &sys_config_str).context("Failed to write sys config")?; + let simulator_config = simulator_config_for_manifest(&config.cvm, &manifest)?; + sync_tee_simulator_config( + &vm_work_dir.shared_dir(), + simulator_config.as_ref(), + &sys_config_str, + )?; // Create vm-state.json with initial state vm_work_dir @@ -281,8 +279,24 @@ Compose file content (first 200 chars): gateway_enabled: app_compose.gateway_enabled(), }; + let mut runtime_networks = resolved_networks(&manifest, &config.cvm); + // Settle the data plane before anything reads it, so `vhost_enabled()` is a + // decision rather than a request and the launch does not warn about a + // `/dev/vhost-net` the netdev it then builds never opens. + settle_vhost(&mut runtime_networks); + // Bridge and macvtap host interfaces belong to netd, whose lifecycle + // one-shot does not manage. Refusing is the honest answer: the alternative + // was to quietly build a different interface here than the server would, + // and then report the VM as if it had the one it asked for. + if !dry_run && runtime_networks.iter().any(needs_netd_interface) { + anyhow::bail!( + "one-shot execution does not manage netd interface lifecycle, which bridge and \ + macvtap networking need; run the VMM server directly or use --dry-run" + ); + } + let process_configs = vm_builder_config - .config_qemu(&workdir_path, &config.cvm, &gpus) + .config_qemu(&workdir_path, &config.cvm, &gpus, &runtime_networks) .context("Failed to build QEMU configuration")?; // Get the main QEMU process config (first in the list) @@ -312,8 +326,8 @@ Compose file content (first 200 chars): } else { println!("# Executing QEMU..."); - // Change working directory to match supervisor process behavior - std::env::set_current_dir(&workdir_path).context("Failed to change working directory")?; + crate::gpu_reset::sanitize_on_attach(&config.cvm.gpu, &gpus) + .context("Failed to sanitize GPUs before QEMU launch")?; let mut cmd = std::process::Command::new(&process_config.command); cmd.args(&process_config.args); @@ -325,14 +339,14 @@ Compose file content (first 200 chars): // Configure stdio to match supervisor behavior if !process_config.stdout.is_empty() { - let stdout_file = std::fs::File::create(&process_config.stdout) - .context("Failed to create stdout file")?; - cmd.stdout(stdout_file); + let stdout_file = + fs::File::create(&process_config.stdout).context("Failed to create stdout file")?; + cmd.stdout(stdout_file.into_file()); } if !process_config.stderr.is_empty() { - let stderr_file = std::fs::File::create(&process_config.stderr) - .context("Failed to create stderr file")?; - cmd.stderr(stderr_file); + let stderr_file = + fs::File::create(&process_config.stderr).context("Failed to create stderr file")?; + cmd.stderr(stderr_file.into_file()); } cmd.current_dir(&workdir_path); @@ -365,7 +379,7 @@ Compose file content (first 200 chars): } eprintln!("# Try running with --dry-run to check the generated command"); - std::process::exit(status.code().unwrap_or(1)); + anyhow::bail!("QEMU exited with status: {status}"); } } diff --git a/dstack/vmm/src/tests/test_vmm_cli.py b/dstack/vmm/src/tests/test_vmm_cli.py new file mode 100644 index 000000000..f3c33c371 --- /dev/null +++ b/dstack/vmm/src/tests/test_vmm_cli.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +"""Focused regression tests for vmm-cli update request construction.""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import sys +import unittest +from pathlib import Path + + +def load_vmm_cli(): + """Load the executable vmm-cli.py as a normal Python module.""" + path = Path(__file__).resolve().parents[1] / "vmm-cli.py" + spec = importlib.util.spec_from_file_location("dstack_vmm_cli", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +VMM_CLI = load_vmm_cli() + + +class UpdateVmTests(unittest.TestCase): + """Verify update flags are not silently discarded by the CLI.""" + + def test_kms_urls_are_sent_to_upgrade_app(self) -> None: + """Send both the update flag and requested KMS URL list.""" + cli = VMM_CLI.VmmCLI("http://127.0.0.1:8080") + calls: list[tuple[str, dict]] = [] + cli.rpc_call = lambda method, params=None: calls.append((method, params)) or {} + + with contextlib.redirect_stdout(io.StringIO()): + cli.update_vm("vm-id", kms_urls=["https://kms.example:8000"]) + + self.assertEqual( + calls, + [ + ( + "UpgradeApp", + { + "id": "vm-id", + "update_kms_urls": True, + "kms_urls": ["https://kms.example:8000"], + }, + ) + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/dstack/vmm/src/vm_launcher.rs b/dstack/vmm/src/vm_launcher.rs new file mode 100644 index 000000000..001d02018 --- /dev/null +++ b/dstack/vmm/src/vm_launcher.rs @@ -0,0 +1,525 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{bail, Context, Result}; +use nix::{ + errno::Errno, + fcntl::{fcntl, FcntlArg, FdFlag}, + sys::{ + prctl, + signal::{kill, raise, Signal}, + }, + unistd::{dup2, getpid, getppid, setpgid, Pid}, +}; +use serde::{Deserialize, Serialize}; +use tokio::process::{Child, Command}; +use tokio::signal::unix::{signal, SignalKind}; +use tokio::time::{sleep, timeout, Instant}; +use tracing::{error, info, warn}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ChildCommand { + pub command: String, + #[serde(default)] + pub args: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LaunchSpec { + pub qemu: ChildCommand, + #[serde(default)] + pub swtpm: Option, + #[serde(default)] + pub swtpm_socket: Option, + #[serde(default)] + pub open_files: Vec, + #[serde(default = "default_startup_timeout_ms")] + pub startup_timeout_ms: u64, + #[serde(default = "default_shutdown_timeout_ms")] + pub shutdown_timeout_ms: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OpenFile { + pub fd: i32, + pub path: PathBuf, +} + +fn default_startup_timeout_ms() -> u64 { + 5_000 +} + +fn default_shutdown_timeout_ms() -> u64 { + 10_000 +} + +struct SocketCleanup(PathBuf); + +impl Drop for SocketCleanup { + fn drop(&mut self) { + if self.0.exists() { + if let Err(error) = fs_err::remove_file(&self.0) { + warn!(path = %self.0.display(), %error, "failed to clean up swtpm socket"); + } + } + } +} + +impl LaunchSpec { + fn is_single_process(&self) -> bool { + self.swtpm.is_none() + } +} + +struct PreparedOpenFiles { + // Retain the original device handles through the fork or exec handoff; + // they close automatically when this prepared set leaves scope. + _opened: Vec, + // Own the collision-free source fds used by dup2. Dropping these before + // fork or exec would close the source fds while `mappings` still refers to them. + _inherited: Vec, + mappings: Vec<(i32, i32)>, +} + +fn prepare_open_files(open_files: &[OpenFile]) -> Result { + let opened = open_files + .iter() + .map(|file| { + fs_err::OpenOptions::new() + .read(true) + .write(true) + .open(&file.path) + .with_context(|| format!("failed to open {}", file.path.display())) + }) + .collect::>>()?; + let max_target = open_files.iter().map(|file| file.fd).max().unwrap_or(2); + let inherited = opened + .iter() + .map(|file| { + let fd = fcntl(file.as_raw_fd(), FcntlArg::F_DUPFD_CLOEXEC(max_target + 1)) + .context("failed to reserve inherited file descriptor")?; + // SAFETY: F_DUPFD_CLOEXEC returned a new descriptor owned by this process. + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) + }) + .collect::>>()?; + let mappings = open_files + .iter() + .zip(&inherited) + .map(|(file, opened)| (file.fd, opened.as_raw_fd())) + .collect::>(); + Ok(PreparedOpenFiles { + _opened: opened, + _inherited: inherited, + mappings, + }) +} + +fn prepare_child(expected_parent: libc::pid_t, mappings: &[(i32, i32)]) -> std::io::Result<()> { + setpgid(Pid::from_raw(0), Pid::from_raw(0))?; + prctl::set_pdeathsig(Signal::SIGKILL)?; + if getppid().as_raw() != expected_parent { + raise(Signal::SIGKILL)?; + } + for &(target, source) in mappings { + if target < 3 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "open file target fd must be at least 3", + )); + } + if source == target { + fcntl(target, FcntlArg::F_SETFD(FdFlag::empty()))?; + } else { + dup2(source, target)?; + } + } + Ok(()) +} + +fn spawn_child(spec: &ChildCommand, open_files: &[OpenFile]) -> Result { + let parent = getpid().as_raw(); + let mut command = Command::new(&spec.command); + command.args(&spec.args); + let prepared = prepare_open_files(open_files)?; + let mappings = prepared.mappings.clone(); + // SAFETY: pre_exec only invokes async-signal-safe libc operations. Checking + // the parent after PR_SET_PDEATHSIG closes the fork/parent-exit race. + unsafe { + command + .as_std_mut() + .pre_exec(move || prepare_child(parent, &mappings)); + } + command + .spawn() + .with_context(|| format!("failed to start {}", spec.command)) +} + +fn exec_in_place(spec: &ChildCommand, open_files: &[OpenFile]) -> Result<()> { + let prepared = prepare_open_files(open_files)?; + let parent = getppid().as_raw(); + prepare_child(parent, &prepared.mappings).context("failed to prepare in-place QEMU exec")?; + let mut command = std::process::Command::new(&spec.command); + command.args(&spec.args); + let error = command.exec(); + Err(error).with_context(|| format!("failed to exec {}", spec.command)) +} + +async fn stop_child(child: &mut Child, name: &str, grace: Duration) { + let Some(pid) = child.id() else { + return; + }; + if let Err(error) = kill(Pid::from_raw(-(pid as libc::pid_t)), Signal::SIGTERM) { + if error != Errno::ESRCH { + warn!(%pid, %name, %error, "failed to terminate child"); + } + } + match timeout(grace, child.wait()).await { + Ok(Ok(status)) => info!(%pid, %name, %status, "child stopped"), + Ok(Err(error)) => warn!(%pid, %name, %error, "failed to wait for child"), + Err(_) => { + warn!(%pid, %name, "child did not stop gracefully; killing"); + if let Err(error) = kill(Pid::from_raw(-(pid as libc::pid_t)), Signal::SIGKILL) { + if error != Errno::ESRCH { + warn!(%pid, %name, %error, "failed to kill child process group"); + } + } + if let Err(error) = child.wait().await { + warn!(%pid, %name, %error, "failed to reap killed child"); + } + } + } +} + +async fn wait_for_swtpm(swtpm: &mut Child, socket: &Path, deadline: Instant) -> Result<()> { + loop { + if socket.exists() { + return Ok(()); + } + if Instant::now() >= deadline { + bail!("timed out waiting for swtpm socket {}", socket.display()); + } + tokio::select! { + status = swtpm.wait() => { + bail!("swtpm exited before socket was ready: {}", status?); + } + _ = sleep(Duration::from_millis(50)) => {} + } + } +} + +pub async fn run(spec_path: &Path) -> Result<()> { + let raw = fs_err::read(spec_path) + .with_context(|| format!("failed to read launch spec {}", spec_path.display()))?; + let spec: LaunchSpec = serde_json::from_slice(&raw).context("failed to parse launch spec")?; + if spec.is_single_process() { + return exec_in_place(&spec.qemu, &spec.open_files); + } + let _socket_cleanup = spec.swtpm_socket.clone().map(SocketCleanup); + if let Some(socket) = &spec.swtpm_socket { + if socket.exists() { + fs_err::remove_file(socket).context("failed to remove stale swtpm socket")?; + } + } + + let grace = Duration::from_millis(spec.shutdown_timeout_ms); + let mut terminate = signal(SignalKind::terminate()).context("failed to watch SIGTERM")?; + let mut interrupt = signal(SignalKind::interrupt()).context("failed to watch SIGINT")?; + let mut swtpm = if let Some(command) = &spec.swtpm { + Some(spawn_child(command, &[])?) + } else { + None + }; + enum StartupExit { + Ready, + Error(anyhow::Error), + Signal, + } + let startup_exit = if let (Some(swtpm), Some(socket)) = (&mut swtpm, &spec.swtpm_socket) { + let startup = wait_for_swtpm( + swtpm, + socket, + Instant::now() + Duration::from_millis(spec.startup_timeout_ms), + ); + tokio::pin!(startup); + tokio::select! { + result = &mut startup => match result { + Ok(()) => StartupExit::Ready, + Err(error) => StartupExit::Error(error), + }, + _ = terminate.recv() => StartupExit::Signal, + _ = interrupt.recv() => StartupExit::Signal, + } + } else { + StartupExit::Ready + }; + match startup_exit { + StartupExit::Ready => {} + StartupExit::Error(error) => { + if let Some(child) = &mut swtpm { + stop_child(child, "swtpm", grace).await; + } + return Err(error); + } + StartupExit::Signal => { + if let Some(child) = &mut swtpm { + stop_child(child, "swtpm", grace).await; + } + return Ok(()); + } + } + + let mut qemu = match spawn_child(&spec.qemu, &spec.open_files) { + Ok(child) => child, + Err(error) => { + if let Some(child) = &mut swtpm { + stop_child(child, "swtpm", grace).await; + } + return Err(error); + } + }; + info!( + qemu_pid = qemu.id(), + swtpm_pid = swtpm.as_ref().and_then(|child| child.id()), + "VM processes started" + ); + + enum Exit { + Signal, + Qemu(std::process::ExitStatus), + Swtpm(std::process::ExitStatus), + } + let exit = tokio::select! { + _ = terminate.recv() => Exit::Signal, + _ = interrupt.recv() => Exit::Signal, + status = qemu.wait() => Exit::Qemu(status.context("failed to wait for QEMU")?), + status = async { + match &mut swtpm { + Some(child) => child.wait().await, + None => std::future::pending().await, + } + } => Exit::Swtpm(status.context("failed to wait for swtpm")?), + }; + + match exit { + Exit::Signal => { + stop_child(&mut qemu, "qemu", grace).await; + if let Some(child) = &mut swtpm { + stop_child(child, "swtpm", grace).await; + } + Ok(()) + } + Exit::Qemu(status) => { + if let Some(child) = &mut swtpm { + stop_child(child, "swtpm", grace).await; + } + if status.success() { + Ok(()) + } else { + bail!("QEMU exited with {status}") + } + } + Exit::Swtpm(status) => { + error!(%status, "swtpm exited while QEMU was running"); + stop_child(&mut qemu, "qemu", grace).await; + bail!("swtpm exited with {status}") + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::net::UnixListener; + + fn shell(script: String) -> ChildCommand { + ChildCommand { + command: "/bin/sh".into(), + args: vec!["-c".into(), script], + } + } + + fn process_is_gone(pid: libc::pid_t) -> bool { + matches!(kill(Pid::from_raw(pid), None), Err(Errno::ESRCH)) + } + + async fn create_fake_socket(path: PathBuf) { + sleep(Duration::from_millis(100)).await; + let _listener = UnixListener::bind(path).unwrap(); + sleep(Duration::from_secs(5)).await; + } + + #[tokio::test] + async fn passes_open_file_to_child() -> Result<()> { + let dir = tempfile::tempdir()?; + let input = dir.path().join("input"); + let output = dir.path().join("output"); + fs_err::write(&input, b"macvtap-fd")?; + let command = shell(format!("cat <&3 > {}", output.display())); + let open_files = vec![OpenFile { fd: 3, path: input }]; + let mut child = spawn_child(&command, &open_files)?; + assert!(child.wait().await?.success()); + + assert_eq!(fs_err::read(output)?, b"macvtap-fd"); + Ok(()) + } + + #[test] + fn single_process_launch_omits_swtpm() { + let mut spec = LaunchSpec { + qemu: shell("exit 0".into()), + swtpm: None, + swtpm_socket: None, + open_files: vec![], + startup_timeout_ms: 1_000, + shutdown_timeout_ms: 1_000, + }; + assert!(spec.is_single_process()); + spec.swtpm = Some(shell("exit 0".into())); + assert!(!spec.is_single_process()); + } + + #[tokio::test] + async fn qemu_failure_stops_and_reaps_swtpm() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("swtpm.sock"); + let swtpm_pid = dir.path().join("swtpm.pid"); + let spec = LaunchSpec { + qemu: shell("exit 7".into()), + swtpm: Some(shell(format!( + "echo $$ > {}; sleep 30", + swtpm_pid.display() + ))), + swtpm_socket: Some(socket.clone()), + open_files: vec![], + startup_timeout_ms: 2_000, + shutdown_timeout_ms: 500, + }; + let spec_path = dir.path().join("spec.json"); + fs_err::write(&spec_path, serde_json::to_vec(&spec).unwrap()).unwrap(); + tokio::spawn(create_fake_socket(socket)); + + assert!(run(&spec_path).await.is_err()); + let pid: libc::pid_t = fs_err::read_to_string(swtpm_pid) + .unwrap() + .trim() + .parse() + .unwrap(); + assert!(process_is_gone(pid)); + } + + #[tokio::test] + async fn swtpm_failure_stops_and_reaps_qemu() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("swtpm.sock"); + let qemu_pid = dir.path().join("qemu.pid"); + let spec = LaunchSpec { + qemu: shell(format!("echo $$ > {}; sleep 30", qemu_pid.display())), + swtpm: Some(shell("sleep 0.2; exit 9".into())), + swtpm_socket: Some(socket.clone()), + open_files: vec![], + startup_timeout_ms: 2_000, + shutdown_timeout_ms: 500, + }; + let spec_path = dir.path().join("spec.json"); + fs_err::write(&spec_path, serde_json::to_vec(&spec).unwrap()).unwrap(); + tokio::spawn(create_fake_socket(socket)); + + assert!(run(&spec_path).await.is_err()); + let pid: libc::pid_t = fs_err::read_to_string(qemu_pid) + .unwrap() + .trim() + .parse() + .unwrap(); + assert!(process_is_gone(pid)); + } + + #[tokio::test] + async fn startup_timeout_stops_swtpm_and_removes_socket() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("swtpm.sock"); + let swtpm_pid = dir.path().join("swtpm.pid"); + let spec = LaunchSpec { + qemu: shell("exit 0".into()), + swtpm: Some(shell(format!( + "echo $$ > {}; sleep 30", + swtpm_pid.display() + ))), + swtpm_socket: Some(socket.clone()), + open_files: vec![], + startup_timeout_ms: 100, + shutdown_timeout_ms: 500, + }; + let spec_path = dir.path().join("spec.json"); + fs_err::write(&spec_path, serde_json::to_vec(&spec).unwrap()).unwrap(); + + assert!(run(&spec_path).await.is_err()); + let pid: libc::pid_t = fs_err::read_to_string(swtpm_pid) + .unwrap() + .trim() + .parse() + .unwrap(); + assert!(process_is_gone(pid)); + assert!(!socket.exists()); + } + + #[tokio::test] + async fn successful_qemu_exit_stops_swtpm_and_cleans_socket() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("swtpm.sock"); + let swtpm_pid = dir.path().join("swtpm.pid"); + let spec = LaunchSpec { + qemu: shell("sleep 0.1; exit 0".into()), + swtpm: Some(shell(format!( + "echo $$ > {}; sleep 30", + swtpm_pid.display() + ))), + swtpm_socket: Some(socket.clone()), + open_files: vec![], + startup_timeout_ms: 2_000, + shutdown_timeout_ms: 500, + }; + let spec_path = dir.path().join("spec.json"); + fs_err::write(&spec_path, serde_json::to_vec(&spec).unwrap()).unwrap(); + tokio::spawn(create_fake_socket(socket.clone())); + + assert!(run(&spec_path).await.is_ok()); + let pid: libc::pid_t = fs_err::read_to_string(swtpm_pid) + .unwrap() + .trim() + .parse() + .unwrap(); + assert!(process_is_gone(pid)); + assert!(!socket.exists()); + } + + #[tokio::test] + async fn swtpm_spawn_failure_removes_stale_socket_without_starting_qemu() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("swtpm.sock"); + fs_err::write(&socket, b"stale").unwrap(); + let qemu_marker = dir.path().join("qemu-started"); + let spec = LaunchSpec { + qemu: shell(format!("touch {}", qemu_marker.display())), + swtpm: Some(ChildCommand { + command: dir.path().join("missing-swtpm").display().to_string(), + args: vec![], + }), + swtpm_socket: Some(socket.clone()), + open_files: vec![], + startup_timeout_ms: 100, + shutdown_timeout_ms: 100, + }; + let spec_path = dir.path().join("spec.json"); + fs_err::write(&spec_path, serde_json::to_vec(&spec).unwrap()).unwrap(); + + assert!(run(&spec_path).await.is_err()); + assert!(!socket.exists()); + assert!(!qemu_marker.exists()); + } +} diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 290ee2440..41674726c 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -36,34 +36,74 @@ # VMM discovery directories -# Each user's instances are in $XDG_RUNTIME_DIR/dstack-vmm (typically /run/user//dstack-vmm). -# CLI scans all users' directories so operators can see every instance on the host. +# A VMM registers in $XDG_RUNTIME_DIR/dstack-vmm, falling back to +# /run/user//dstack-vmm when that variable is unset. +# Scanning /run/user is what lets an operator see every user's instances, and +# covers the usual case where the variable holds exactly that path. It does not +# cover a session that points XDG_RUNTIME_DIR somewhere else, so this VMM's own +# directory is added by name rather than assumed to be under /run/user. def _get_discovery_dirs() -> List[Tuple[str, Optional[str]]]: """Return list of (discovery_dir, username) tuples.""" import pwd dirs = [] + seen = set() + + def add(path: str, username: Optional[str]): + if not os.path.isdir(path): + return + key = os.path.realpath(path) + if key in seen: + return + seen.add(key) + dirs.append((path, username)) + run_user = "/run/user" if os.path.isdir(run_user): try: for uid_str in os.listdir(run_user): candidate = os.path.join(run_user, uid_str, "dstack-vmm") - if os.path.isdir(candidate): - try: - username = pwd.getpwuid(int(uid_str)).pw_name - except (KeyError, ValueError): - username = f"uid:{uid_str}" - dirs.append((candidate, username)) + try: + username = pwd.getpwuid(int(uid_str)).pw_name + except (KeyError, ValueError): + username = f"uid:{uid_str}" + add(candidate, username) except PermissionError: pass + + xdg = os.environ.get("XDG_RUNTIME_DIR") + if xdg: + try: + username = pwd.getpwuid(os.getuid()).pw_name + except KeyError: + username = f"uid:{os.getuid()}" + add(os.path.join(xdg, "dstack-vmm"), username) + return dirs +def _discovery_dirs_scanned() -> List[str]: + """Where a listing looked, for a listing that found nothing. + + An empty result means no directory exists yet, so naming the ones that + would have been read beats naming none of them. + """ + found = [d for d, _ in _get_discovery_dirs()] + if found: + return found + candidates = ["/run/user/*/dstack-vmm"] + xdg = os.environ.get("XDG_RUNTIME_DIR") + if xdg: + candidates.append(os.path.join(xdg, "dstack-vmm")) + return candidates + + def load_config() -> Dict[str, Any]: """Load configuration from the default config file. Returns: - Dictionary with configuration values (url, auth_user, auth_password) + Dictionary with configuration values (url, auth_user, auth_password, + auth_token) """ if not os.path.exists(DEFAULT_CONFIG_PATH): @@ -182,9 +222,7 @@ def cmd_ls_vmm(args): if not instances: print("No running VMM instances found.") - print( - f" (scanned: {', '.join(d for d, _ in _get_discovery_dirs()) or '/run/user/*/dstack-vmm'})" - ) + print(f" (scanned: {', '.join(_discovery_dirs_scanned())})") return if getattr(args, "json", False): @@ -320,17 +358,30 @@ def encrypt_env(envs, hex_public_key: str) -> str: def parse_port_mapping(port_str: str) -> Dict: - """Parse a port mapping string into a dictionary.""" + """Parse a port mapping string into a dictionary. + + Accepts an optional "@" suffix naming which NIC the traffic enters + through. Without it the VMM picks: the first user-mode NIC, else the first + bridge NIC. A single-NIC VM never needs it. + """ + nic_index = None + if "@" in port_str: + port_str, _, nic = port_str.rpartition("@") + # `int()` alone would take "1_0" as 10, " 1" as 1, and "+1" as 1. A NIC + # index is a position in a list the user wrote, so only digits are it. + if not (nic.isascii() and nic.isdigit()): + raise argparse.ArgumentTypeError(f"Invalid NIC index: {nic}") + nic_index = int(nic) parts = port_str.split(":") if len(parts) == 3: - return { + mapping = { "protocol": parts[0], "host_address": "127.0.0.1", "host_port": int(parts[1]), "vm_port": int(parts[2]), } elif len(parts) == 4: - return { + mapping = { "protocol": parts[0], "host_address": parts[1], "host_port": int(parts[2]), @@ -338,6 +389,9 @@ def parse_port_mapping(port_str: str) -> Dict: } else: raise argparse.ArgumentTypeError(f"Invalid port mapping format: {port_str}") + if nic_index is not None: + mapping["nic_index"] = nic_index + return mapping def read_utf8(filepath: str) -> str: @@ -371,12 +425,29 @@ def __init__( base_url: str, auth_user: Optional[str] = None, auth_password: Optional[str] = None, + auth_token: Optional[str] = None, ): - """Initialize the client with a base URL and optional auth credentials.""" + """Initialize the client with a base URL and optional auth credentials. + + Two credential forms are supported for a VMM with `[auth]` enabled: + a bearer token (sent as `Authorization: Bearer `), or HTTP Basic + (user + password). A bearer token takes precedence when both are set. + """ self.base_url = base_url.rstrip("/") self.use_uds = self.base_url.startswith("unix:") - self.auth_user = auth_user - self.auth_password = auth_password + self.auth_user = auth_user or None + self.auth_password = auth_password or None + self.auth_token = auth_token or None + # fail fast on a half-configured Basic credential: silently sending no + # auth header would make requests fail against `[auth] enabled = true` + # in a way that is hard to diagnose. + if not self.auth_token and bool(self.auth_user) != bool(self.auth_password): + missing = "password" if self.auth_user else "username" + raise ValueError( + f"incomplete VMM Basic auth credentials: {missing} is missing. " + "set both username and password, or use a bearer token " + "(--token / DSTACK_VMM_TOKEN)." + ) if self.use_uds: self.uds_path = self.base_url[5:] # Remove 'unix:' prefix @@ -410,13 +481,17 @@ def request( if headers is None: headers = {} - # Add Basic Authentication header if credentials are provided - if self.auth_user and self.auth_password: - credentials = f"{self.auth_user}:{self.auth_password}" - encoded_credentials = base64.b64encode(credentials.encode("utf-8")).decode( - "ascii" - ) - headers["Authorization"] = f"Basic {encoded_credentials}" + # Add an auth header when credentials are provided. A bearer token wins + # over Basic when both happen to be set. + if "Authorization" not in headers: + if self.auth_token: + headers["Authorization"] = f"Bearer {self.auth_token}" + elif self.auth_user and self.auth_password: + credentials = f"{self.auth_user}:{self.auth_password}" + encoded_credentials = base64.b64encode( + credentials.encode("utf-8") + ).decode("ascii") + headers["Authorization"] = f"Basic {encoded_credentials}" # Prepare the body if isinstance(body, dict): @@ -480,11 +555,12 @@ def __init__( base_url: str, auth_user: Optional[str] = None, auth_password: Optional[str] = None, + auth_token: Optional[str] = None, ): """Initialize the CLI with a base URL and optional auth credentials.""" self.base_url = base_url.rstrip("/") self.headers = {"Content-Type": "application/json"} - self.client = VmmClient(base_url, auth_user, auth_password) + self.client = VmmClient(base_url, auth_user, auth_password, auth_token) def rpc_call(self, method: str, params: Optional[Dict] = None) -> Dict: """Make an RPC call to the dstack-vmm API.""" @@ -818,6 +894,10 @@ def create_app_compose(self, args) -> None: app_compose["swap_size"] = swap_bytes else: app_compose.pop("swap_size", None) + if args.event_log_version == 2: + app_compose["event_log_version"] = args.event_log_version + elif args.event_log_version == 1: + app_compose.pop("event_log_version", None) compose_file = json.dumps(app_compose, indent=4, ensure_ascii=False).encode( "utf-8" @@ -874,6 +954,8 @@ def create_vm(self, args) -> None: "stopped": args.stopped, "no_tee": args.no_tee, } + if args.simulated_tee: + params["simulated_tee"] = args.simulated_tee if args.swap is not None: swap_bytes = max(0, int(round(args.swap)) * 1024 * 1024) if swap_bytes > 0: @@ -890,15 +972,26 @@ def create_vm(self, args) -> None: params["kms_urls"] = args.kms_url if args.gateway_url: params["gateway_urls"] = args.gateway_url - if args.net: - params["networking"] = {"mode": args.net} + # "auto" is what a fresh deployment already does, so it only means + # something to `update`, where it clears a pinned count. + net_queues = None if args.net_queues == "auto" else args.net_queues + if args.net or args.net_vhost is not None or net_queues: + networking = {} + if args.net: + networking["mode"] = args.net + if args.net_vhost is not None: + networking["vhost"] = args.net_vhost + if net_queues: + networking["queues"] = net_queues + params["networking"] = networking app_id = args.app_id or self.calc_app_id(compose_content) print(f"App ID: {app_id}") if envs: - encrypt_pubkey = self.get_app_env_encrypt_pub_key( - app_id, args.kms_url[0] if args.kms_url else None + encrypt_url = args.kms_encrypt_url or ( + args.kms_url[0] if args.kms_url else None ) + encrypt_pubkey = self.get_app_env_encrypt_pub_key(app_id, encrypt_url) print(f"Encrypting environment variables with key: {encrypt_pubkey}") envs_list = [{"key": k, "value": v} for k, v in envs.items()] params["encrypted_env"] = encrypt_env(envs_list, encrypt_pubkey) @@ -1000,6 +1093,10 @@ def update_vm( no_gpus: bool = False, kms_urls: Optional[List[str]] = None, no_tee: Optional[bool] = None, + net: Optional[str] = None, + net_vhost: Optional[bool] = None, + net_vhost_inherit: bool = False, + net_queues: Optional[Union[int, str]] = None, ) -> None: """Update multiple aspects of a VM in one command.""" # Validate: --env-file requires --kms-url @@ -1123,10 +1220,84 @@ def update_vm( app_compose, indent=4, ensure_ascii=False ) + if net or net_vhost is not None or net_vhost_inherit or net_queues: + # The RPC replaces the whole NIC list, so merge into what the VM + # already has rather than silently dropping its other interfaces or + # un-pinning a bridge it was deployed with. + if vm_info_response is None: + vm_info_response = self.rpc_call("GetInfo", {"id": vm_id}) + if not vm_info_response.get("found", False): + raise Exception(f"VM with ID {vm_id} not found") + configuration = vm_info_response["info"].get("configuration") or {} + current = configuration.get("networks") or [] + if not current and configuration.get("networking"): + current = [configuration["networking"]] + if len(current) > 1: + raise Exception( + "this VM has multiple network interfaces; edit them through the " + "web UI or the UpgradeApp API rather than these flags" + ) + # Only the fields the deployment RPC accepts back travel with the + # update. macvtap_mode is node-controlled and can never be changed, + # so resending it can only fail if the node changed meanwhile. An + # empty mode is meaningful: it says the VM never named a backend + # and still follows the node's. + source = current[0] if current else {} + networking = { + key: source[key] + for key in ("mode", "bridge_name", "parent", "vhost", "queues") + if source.get(key) not in (None, "") + } + if net == "default": + # The only way back to "whatever backend the node runs". Without + # it a VM that named a mode once is pinned to it for life, since + # the merge above carries the reported mode forward on every + # later update. The data plane keeps whatever it was told. + networking.pop("mode", None) + networking.pop("bridge_name", None) + networking.pop("parent", None) + elif net: + networking["mode"] = net + # A field belongs to the mode that owns it. Carrying a bridge into + # a macvtap request, or a parent into a bridge one, asks the server + # about a field the caller never typed and has no flag to clear. + mode = networking.get("mode", "") + if mode and mode != "bridge": + networking.pop("bridge_name", None) + if mode and mode != "macvtap": + networking.pop("parent", None) + # Same rule for the data plane. User networking has neither a vhost + # backend nor multiple queues, so carrying an inherited pin into it + # is rejected for a flag the operator never typed -- and the two + # flags that would clear it are the ones they have not found yet. + # An explicitly typed value still earns the error: that one is + # theirs to be wrong about. + if mode == "user": + if net_vhost is None: + networking.pop("vhost", None) + if not net_queues: + networking.pop("queues", None) + if net_vhost is not None: + networking["vhost"] = net_vhost + elif net_vhost_inherit: + networking.pop("vhost", None) + if net_queues == "auto": + networking.pop("queues", None) + elif net_queues: + networking["queues"] = net_queues + upgrade_params["update_networking"] = True + upgrade_params["networks"] = [networking] + updates.append(f"networking ({networking})") + if user_config: upgrade_params["user_config"] = user_config updates.append("user config") + if kms_urls is not None: + upgrade_params["update_kms_urls"] = True + upgrade_params["kms_urls"] = kms_urls + updates.append(f"KMS URLs ({len(kms_urls)})") + # handle port updates - only update if --port or --no-ports is specified if no_ports or ports is not None: if no_ports: @@ -1212,6 +1383,38 @@ def show_info(self, vm_id: str, json_output: bool = False) -> None: if info.get("shutdown_progress"): print(f"Shutdown: {info['shutdown_progress']}") + interfaces = info.get("interfaces") or [] + if interfaces: + print("\nNetwork Interfaces:") + for iface in interfaces: + parts = [ + f"{iface.get('netdev_id') or '-':<6}", + f"{iface.get('mode') or '-'}/{iface.get('backend') or '-'}", + iface.get("mac") or "-", + ] + if iface.get("bridge_name"): + parts.append(f"bridge={iface['bridge_name']}") + if iface.get("macvtap_mode"): + parts.append(f"macvtap_mode={iface['macvtap_mode']}") + # Absent, not false: custom mode carries an operator-written + # netdev string the VMM never parses, so it reports no data + # plane rather than asserting the resolved default over one that + # may well say vhost=on,queues=8. + vhost = iface.get("vhost") + parts.append( + "vhost=" + ("-" if vhost is None else ("on" if vhost else "off")) + ) + parts.append(f"queues={iface.get('queues') or '-'}") + print(" " + " ".join(parts)) + # A stopped VM has no interfaces to describe, so these are what its + # next launch would build -- which can differ from its last one. + # + # The server's own predicate, not the status string: a VM being + # removed with QEMU still up reports the interfaces that process + # built, and no status value says so. + if not info.get("running", False): + print(" (not running; shown as its next launch would build them)") + events = info.get("events", []) if events: print("\nRecent Events:") @@ -1501,6 +1704,26 @@ def save_whitelist(whitelist: List[str]) -> None: json.dump({"trusted_signers": whitelist}, f, indent=2) +def queue_count(value: str) -> Union[int, str]: + """Parse a queue pair count the node could act on, or "auto" to stop pinning one. + + Zero would otherwise reach the wire as "unset" and be answered with the + default, and a negative one as a decoding error naming a column offset -- + neither of which tells the caller what they asked for was impossible. + """ + if value == "auto": + return "auto" + try: + count = int(value) + except ValueError: + raise argparse.ArgumentTypeError(f"'{value}' is not a whole number or 'auto'") + if count < 1: + raise argparse.ArgumentTypeError( + f"queue pairs must be at least 1, or 'auto' to follow the vCPU count; got {count}" + ) + return count + + def main(): """Parse arguments and dispatch to the appropriate command handler.""" parser = argparse.ArgumentParser(description="dstack-vmm CLI - Manage VMs") @@ -1519,6 +1742,7 @@ def main(): default_auth_password = os.environ.get( "DSTACK_VMM_AUTH_PASSWORD", config.get("auth_password") ) + default_auth_token = os.environ.get("DSTACK_VMM_TOKEN", config.get("auth_token")) parser.add_argument( "--url", @@ -1537,6 +1761,13 @@ def main(): default=default_auth_password, help="Basic auth password (can also be set via DSTACK_VMM_AUTH_PASSWORD env var or config file)", ) + parser.add_argument( + "--token", + default=default_auth_token, + help="bearer token for a VMM with `[auth]` enabled, sent as " + "`Authorization: Bearer` (can also be set via DSTACK_VMM_TOKEN env var " + "or config file). Takes precedence over --auth-user/--auth-password.", + ) subparsers = parser.add_subparsers(dest="command", help="Commands") @@ -1684,6 +1915,13 @@ def _patched_format_help(): default=None, help="Swap size (e.g. 4G). Set to 0 to disable", ) + compose_parser.add_argument( + "--event-log-version", + type=int, + choices=[1, 2], + default=None, + help="RTMR3 runtime event-log digest format (1: legacy binary, 2: JCS canonical JSON). Omit to use the guest default (1).", + ) compose_parser.add_argument( "--output", required=True, help="Path to output app-compose.json file" ) @@ -1722,7 +1960,7 @@ def _patched_format_help(): "--port", action="append", type=str, - help="Port mapping in format: protocol[:address]:from:to", + help="Port mapping in format: protocol[:address]:from:to[@nic]", ) deploy_parser.add_argument( "--gpu", @@ -1742,6 +1980,11 @@ def _patched_format_help(): "--hugepages", action="store_true", help="Enable hugepages for the VM" ) deploy_parser.add_argument("--kms-url", action="append", type=str, help="KMS URL") + deploy_parser.add_argument( + "--kms-encrypt-url", + type=str, + help="Controller-reachable KMS URL used only to encrypt --env-file", + ) deploy_parser.add_argument( "--gateway-url", action="append", type=str, help="Gateway URL" ) @@ -1763,11 +2006,44 @@ def _patched_format_help(): help="Force-enable Intel TDX (default)", ) deploy_parser.set_defaults(no_tee=False) + deploy_parser.add_argument( + "--simulated-tee", + choices=[ + "dstack-tdx", + "dstack-gcp-tdx", + "dstack-amd-sev-snp", + "dstack-nitro-enclave", + "dstack-aws-nitro-tpm", + ], + help="Simulate the selected TEE ABI for this VM (development images only)", + ) deploy_parser.add_argument( "--net", - choices=["bridge", "user"], + choices=["bridge", "user", "macvtap"], help="Networking mode (default: use global config)", ) + net_vhost = deploy_parser.add_mutually_exclusive_group() + net_vhost.add_argument( + "--net-vhost", + dest="net_vhost", + action="store_true", + default=None, + help="Use the host kernel vhost-net data plane (default: use global config)", + ) + net_vhost.add_argument( + "--net-no-vhost", + dest="net_vhost", + action="store_false", + help="Keep packet processing in the QEMU main loop", + ) + deploy_parser.add_argument( + "--net-queues", + type=queue_count, + metavar="N", + help="virtio-net queue pairs, bounded by the node's max_net_queues. " + "Without --net, the node's own networking mode is kept " + "(default: use global config)", + ) # Images command lsimage_parser = subparsers.add_parser("lsimage", help="List available images") @@ -1840,7 +2116,7 @@ def _patched_format_help(): action="append", type=str, required=True, - help="Port mapping in format: protocol[:address]:from:to (can be used multiple times)", + help="Port mapping in format: protocol[:address]:from:to[@nic] (can be used multiple times)", ) # Update (all-in-one) command @@ -1868,13 +2144,49 @@ def _patched_format_help(): "--env-file", help="File with environment variables to encrypt" ) update_parser.add_argument("--user-config", help="Path to user config file") + update_parser.add_argument( + "--net", + choices=["bridge", "user", "macvtap", "default"], + help=( + "Networking mode (applies from the next boot). 'default' stops " + "pinning a mode and follows the node's, the way --net-queues auto " + "and --net-vhost-default stop pinning the data plane" + ), + ) + update_net_vhost = update_parser.add_mutually_exclusive_group() + update_net_vhost.add_argument( + "--net-vhost", + dest="net_vhost", + action="store_true", + default=None, + help="Use the host kernel vhost-net data plane", + ) + update_net_vhost.add_argument( + "--net-no-vhost", + dest="net_vhost", + action="store_false", + help="Keep packet processing in the QEMU main loop", + ) + update_net_vhost.add_argument( + "--net-vhost-default", + dest="net_vhost_inherit", + action="store_true", + help="Stop pinning vhost and follow the node default again", + ) + update_parser.add_argument( + "--net-queues", + type=queue_count, + metavar="N", + help="virtio-net queue pairs, bounded by the node's max_net_queues. " + "Use 'auto' to stop pinning a count and follow the vCPU count again", + ) # Port mapping options (mutually exclusive with --no-ports) port_group = update_parser.add_mutually_exclusive_group() port_group.add_argument( "--port", action="append", type=str, - help="Port mapping in format: protocol[:address]:from:to (can be used multiple times)", + help="Port mapping in format: protocol[:address]:from:to[@nic] (can be used multiple times)", ) port_group.add_argument( "--no-ports", @@ -1939,7 +2251,11 @@ def _patched_format_help(): # Resolve the URL with auto-discovery url = resolve_vmm_url(instances, config, args.url) - cli = VmmCLI(url, args.auth_user, args.auth_password) + try: + cli = VmmCLI(url, args.auth_user, args.auth_password, args.token) + except ValueError as e: + print(f"error: {e}", file=sys.stderr) + sys.exit(1) if args.command == "lsvm": cli.list_vms(args.verbose, args.json) @@ -2007,6 +2323,10 @@ def _patched_format_help(): no_gpus=args.no_gpus if hasattr(args, "no_gpus") else False, kms_urls=args.kms_url, no_tee=args.no_tee, + net=args.net, + net_vhost=args.net_vhost, + net_vhost_inherit=getattr(args, "net_vhost_inherit", False), + net_queues=args.net_queues, ) elif args.command == "kms": if not args.kms_action: diff --git a/dstack/vmm/ui/README.md b/dstack/vmm/ui/README.md index 8ba28fb0b..0db86913a 100644 --- a/dstack/vmm/ui/README.md +++ b/dstack/vmm/ui/README.md @@ -15,8 +15,8 @@ npm run watch `dstack-vmm` now builds the single-file HTML artifact from `build.rs` and writes it to Cargo's `OUT_DIR`. This requires Node.js and npm to be installed; if they are -missing, the Rust build will fail with an installation hint. The previous -`console_v0.html` remains untouched so the legacy UI stays available under `/v0`. +missing, the Rust build will fail with an installation hint. The legacy +`console_v0.html` route has been removed. The UI codebase is written in TypeScript. The build pipeline performs three steps: diff --git a/dstack/vmm/ui/package-lock.json b/dstack/vmm/ui/package-lock.json index 40def9f69..9c0a860de 100644 --- a/dstack/vmm/ui/package-lock.json +++ b/dstack/vmm/ui/package-lock.json @@ -203,7 +203,9 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "2.0.2", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", "dev": true, "license": "MIT", "dependencies": { @@ -472,9 +474,9 @@ } }, "node_modules/linkify-it": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.1.tgz", - "integrity": "sha512-wVoTjP4Q6R0NW5hiZkVJaFZPWgtXfoGF+6LucL3/FtiNjmcHhYjEr5f1Kqjirc1nBW07J/ZuRFumqr2oqccEWg==", + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.2.tgz", + "integrity": "sha512-ONTm2jCMAVZjgQa/Fy1kScXsuOoF5NPTsoFBdE1KVIZ2vAh/r9+Bqo+0jINCBYnavTPQZz38QzFTme79ENoN3Q==", "dev": true, "funding": [ { diff --git a/dstack/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts index c0e59eb20..4334d8844 100644 --- a/dstack/vmm/ui/src/components/CreateVmDialog.ts +++ b/dstack/vmm/ui/src/components/CreateVmDialog.ts @@ -22,7 +22,44 @@ const CreateVmDialogComponent = { portMappingEnabled: { type: Boolean, required: true }, networkingModes: { type: Array, required: true }, defaultBridge: { type: String, default: '' }, + maxNetQueues: { type: Number, default: 0 }, defaultNetworkingLabel: { type: String, required: true }, + defaultModeTunable: { type: Boolean, default: false }, + defaultVhostOn: { type: Boolean, default: false }, + }, + methods: { + // Whether this NIC will end up on the vhost data plane. An unset select + // means it follows the node, and a node with vhost off gives one queue pair + // however many vCPUs the VM has -- so the answer is not readable from this + // row alone. + vhostOn(network: { vhost?: string }) { + if (network.vhost === 'on') { + return true; + } + if (network.vhost === 'off') { + return false; + } + return (this as any).defaultVhostOn; + }, + // What an empty queues field actually resolves to. It is the vCPU count + // only when vhost is on: with vhost off the backend has no multiqueue data + // plane and the NIC gets exactly one queue pair. + queuesHint(network: { vhost?: string }) { + if (!this.vhostOn(network)) { + return 'virtio-net queue pairs. Empty means one queue pair, because vhost is off.'; + } + const cap = (this as any).maxNetQueues + ? `, capped at ${(this as any).maxNetQueues} on this node` + : ''; + return `virtio-net queue pairs. Empty follows the VM's vCPU count${cap}.`; + }, + queuesPlaceholder(network: { vhost?: string }) { + if (!this.vhostOn(network)) { + return 'queues: auto (1, vhost off)'; + } + const cap = (this as any).maxNetQueues ? ` (max ${(this as any).maxNetQueues})` : ''; + return `queues: auto${cap}`; + }, }, emits: ['close', 'submit', 'load-compose'], template: /* html */ ` @@ -90,6 +127,14 @@ const CreateVmDialogComponent = { +
+ +
+
@@ -108,10 +153,15 @@ const CreateVmDialogComponent = {
-
@@ -145,7 +195,7 @@ const CreateVmDialogComponent = {
-
+
@@ -156,6 +206,7 @@ const CreateVmDialogComponent = {
{{ defaultNetworkingLabel }}
+ + + + + {{ defaultBridge ? 'Leave empty to use the VMM default bridge from vmm.toml: ' + defaultBridge + '.' : 'No default bridge is configured in vmm.toml; enter a bridge interface name.' }} Guest IP is assigned by host DHCP on that bridge and reported after boot. + + Leave empty to use the macvtap parent from vmm.toml. The forwarding mode stays node-controlled. + + + {{ defaultNetworkingLabel }} has no vhost-net or multiqueue data plane, so these two settings are recorded + but stay dormant until this node's default backend can carry them. +
- +
+
+ + +
+
@@ -184,18 +277,34 @@ const CreateVmDialogComponent = { - +
+
+ + + Selecting a platform makes this instance run without hardware TEE. Key provider selection remains independent. +
+
- +
diff --git a/dstack/vmm/ui/src/components/PortMappingEditor.ts b/dstack/vmm/ui/src/components/PortMappingEditor.ts index 0668bd952..70a4ad278 100644 --- a/dstack/vmm/ui/src/components/PortMappingEditor.ts +++ b/dstack/vmm/ui/src/components/PortMappingEditor.ts @@ -8,6 +8,10 @@ type PortEntry = { host_port: number | null; vm_port: number | null; custom_ip?: string; // User-entered IP for custom mode + // Which NIC the traffic enters through. Blank lets the VMM pick: the first + // user-mode NIC, else the first bridge NIC. A single-NIC VM never needs it, + // which is why the field only appears once a VM has more than one. + nic_index?: number | string | null; }; // ... keep your types as-is ... @@ -21,6 +25,9 @@ const PortMappingEditorComponent = { name: 'PortMappingEditor', props: { ports: { type: Array, required: true }, + // How many NICs the VM has. One NIC has nothing to choose between, so the + // column stays hidden rather than offering a pin that can only be 0. + nicCount: { type: Number, default: 1 }, }, // normalize on initial load @@ -63,6 +70,15 @@ const PortMappingEditorComponent = { + @@ -95,6 +111,7 @@ const PortMappingEditorComponent = { custom_ip: '', host_port: null, vm_port: null, + nic_index: null, }); }, diff --git a/dstack/vmm/ui/src/components/UpdateVmDialog.ts b/dstack/vmm/ui/src/components/UpdateVmDialog.ts index f994f0487..4fa40e493 100644 --- a/dstack/vmm/ui/src/components/UpdateVmDialog.ts +++ b/dstack/vmm/ui/src/components/UpdateVmDialog.ts @@ -21,10 +21,47 @@ const UpdateVmDialogComponent = { portMappingEnabled: { type: Boolean, required: true }, networkingModes: { type: Array, required: true }, defaultBridge: { type: String, default: '' }, + maxNetQueues: { type: Number, default: 0 }, defaultNetworkingLabel: { type: String, required: true }, + defaultModeTunable: { type: Boolean, default: false }, + defaultVhostOn: { type: Boolean, default: false }, kmsEnabled: { type: Boolean, required: true }, composeHashPreview: { type: String, required: true }, }, + methods: { + // Whether this NIC will end up on the vhost data plane. An unset select + // means it follows the node, and a node with vhost off gives one queue pair + // however many vCPUs the VM has -- so the answer is not readable from this + // row alone. + vhostOn(network: { vhost?: string }) { + if (network.vhost === 'on') { + return true; + } + if (network.vhost === 'off') { + return false; + } + return (this as any).defaultVhostOn; + }, + // What an empty queues field actually resolves to. It is the vCPU count + // only when vhost is on: with vhost off the backend has no multiqueue data + // plane and the NIC gets exactly one queue pair. + queuesHint(network: { vhost?: string }) { + if (!this.vhostOn(network)) { + return 'virtio-net queue pairs. Empty means one queue pair, because vhost is off.'; + } + const cap = (this as any).maxNetQueues + ? `, capped at ${(this as any).maxNetQueues} on this node` + : ''; + return `virtio-net queue pairs. Empty follows the VM's vCPU count${cap}.`; + }, + queuesPlaceholder(network: { vhost?: string }) { + if (!this.vhostOn(network)) { + return 'queues: auto (1, vhost off)'; + } + const cap = (this as any).maxNetQueues ? ` (max ${(this as any).maxNetQueues})` : ''; + return `queues: auto${cap}`; + }, + }, emits: ['close', 'submit', 'load-compose'], template: /* html */ `
@@ -92,10 +129,15 @@ const UpdateVmDialogComponent = {
-
- +
@@ -145,24 +190,67 @@ const UpdateVmDialogComponent = {
{{ defaultNetworkingLabel }}
+ + + + + + {{ defaultBridge ? 'Leave empty to use the VMM default bridge from vmm.toml: ' + defaultBridge + '.' : 'No default bridge is configured in vmm.toml; enter a bridge interface name.' }} Guest IP is assigned by host DHCP on that bridge and reported after boot. + + Leave empty to use the macvtap parent from vmm.toml. The forwarding mode stays node-controlled. + + + {{ defaultNetworkingLabel }} has no vhost-net or multiqueue data plane, so these two settings are recorded + but stay dormant until this node's default backend can carry them. +
- +
diff --git a/dstack/vmm/ui/src/composables/useVmManager.ts b/dstack/vmm/ui/src/composables/useVmManager.ts index c6c60c741..976a75c7c 100644 --- a/dstack/vmm/ui/src/composables/useVmManager.ts +++ b/dstack/vmm/ui/src/composables/useVmManager.ts @@ -28,12 +28,32 @@ type AppCompose = { secure_time: boolean; requirements?: Requirements; storage_fs?: string; + storage_discard?: boolean; swap_size: number; launch_token_hash?: string; pre_launch_script?: string; - init_script?: string; + init_script?: string | string[]; + event_log_version?: number; }; +function initScriptList(initScript: string | string[] | undefined): string[] { + const scripts = Array.isArray(initScript) + ? [...initScript] + : initScript + ? [initScript] + : []; + return scripts.length > 0 ? scripts : ['']; +} + +function nonEmptyInitScripts(initScripts: string[]): string[] { + return initScripts.filter((script) => script.trim()); +} + +function compatibleInitScripts(initScripts: string[]): string | string[] | undefined { + const scripts = nonEmptyInitScripts(initScripts); + return scripts.length === 0 ? undefined : scripts.length === 1 ? scripts[0] : scripts; +} + type KeyProviderKind = 'none' | 'kms' | 'local' | 'tpm'; type RequirementPlatform = | 'dstack-tdx' @@ -41,7 +61,6 @@ type RequirementPlatform = | 'dstack-amd-sev-snp' | 'dstack-nitro-enclave'; type Requirements = { - os_version?: string; platforms?: RequirementPlatform[]; tdx_measure_acpi_tables?: boolean; launch_token_hash?: string; @@ -74,6 +93,7 @@ type VmListItem = { shutdown_progress?: string; image_version?: string; interfaces?: VmmTypes.INetworkInterfaceStatus[]; + running?: boolean; configuration?: VmConfiguration; appCompose?: AppCompose; }; @@ -88,18 +108,36 @@ type PortFormEntry = { host_address?: string; host_port?: number | null; vm_port?: number | null; + /** + * Which NIC this mapping's traffic enters through. Unset lets the VMM pick. + * Carried through edits unchanged: `GetInfo` reports it and this form sends + * the whole list back, so dropping it here would silently unpin a mapping + * whenever anyone touched an unrelated field. + */ + // `v-model.number` leaves the raw string here when it does not parse, so + // an emptied box is `''` rather than `null`. See `normalizePorts`. + nic_index?: number | string | null; }; type NetworkFormEntry = { mode: string; bridge_name?: string; + /** Pinned at deployment for macvtap NICs; carried through edits unchanged. */ + parent?: string; + /** '' inherits the node default, otherwise 'on' or 'off'. */ + vhost?: string; + /** + * '' lets the queue count follow the vCPU count. A `number` once the operator + * types into the input: `v-model` casts for ``. + */ + queues?: string | number; }; type VmFormState = { name: string; image: string; dockerComposeFile: string; - initScript: string; + initScripts: string[]; preLaunchScript: string; vcpu: number; memory: number; @@ -114,6 +152,7 @@ type VmFormState = { ports: PortFormEntry[]; encryptedEnvs: EncryptedEnvEntry[]; storage_fs: string; + storage_discard: boolean; app_id: string | null; key_provider?: KeyProviderKind; key_provider_id: string; @@ -122,6 +161,7 @@ type VmFormState = { public_sysinfo: boolean; public_tcbinfo: boolean; no_tee: boolean; + simulated_tee: string; pin_numa: boolean; hugepages: boolean; networks: NetworkFormEntry[]; @@ -129,6 +169,7 @@ type VmFormState = { kms_urls: string[]; gateway_urls: string[]; stopped: boolean; + event_log_version: number; }; type UpdateDialogState = { @@ -136,7 +177,7 @@ type UpdateDialogState = { vm: VmListItem | null; updateCompose: boolean; dockerComposeFile: string; - initScript: string; + initScripts: string[]; preLaunchScript: string; encryptedEnvs: EncryptedEnvEntry[]; resetSecrets: boolean; @@ -175,6 +216,7 @@ type CloneConfigDialogState = { hugepages: boolean; pin_numa: boolean; no_tee: boolean; + simulated_tee: string; encrypted_env?: Uint8Array; app_id?: string; stopped: boolean; @@ -185,7 +227,7 @@ function createVmFormState(preLaunchScript: string): VmFormState { name: '', image: '', dockerComposeFile: '', - initScript: '', + initScripts: [''], preLaunchScript, vcpu: 1, memory: 2048, @@ -200,6 +242,7 @@ function createVmFormState(preLaunchScript: string): VmFormState { ports: [], encryptedEnvs: [], storage_fs: '', + storage_discard: true, app_id: null, key_provider: 'kms', key_provider_id: '', @@ -208,6 +251,7 @@ function createVmFormState(preLaunchScript: string): VmFormState { public_sysinfo: true, public_tcbinfo: true, no_tee: false, + simulated_tee: '', pin_numa: false, hugepages: false, networks: [], @@ -215,6 +259,7 @@ function createVmFormState(preLaunchScript: string): VmFormState { kms_urls: [], gateway_urls: [], stopped: false, + event_log_version: 1, }; } @@ -224,7 +269,7 @@ function createUpdateDialogState(): UpdateDialogState { vm: null, updateCompose: false, dockerComposeFile: '', - initScript: '', + initScripts: [''], preLaunchScript: '', encryptedEnvs: [], resetSecrets: false, @@ -265,6 +310,7 @@ function createCloneConfigDialogState(): CloneConfigDialogState { hugepages: false, pin_numa: false, no_tee: false, + simulated_tee: '', encrypted_env: undefined, app_id: undefined, stopped: false, @@ -327,6 +373,18 @@ fi return Array.from(new Set(fallback)); }); const defaultBridge = computed(() => config.value.networking?.default_bridge || ''); + const maxNetQueues = computed(() => config.value.networking?.max_queues || 0); + const defaultVhostOn = computed(() => !!config.value.networking?.default_vhost); + // Whether the node's own default backend can carry vhost-net and multiqueue. + // The RPC accepts the two tuning fields on a mode-less entry regardless -- + // deliberately, so a NIC that inherits its backend stays tunable across a node + // change -- but on a node whose default is user or custom they lie dormant, + // and an operator who sets them deserves to be told that rather than discover + // it in the interfaces panel afterwards. + const defaultModeTunable = computed(() => { + const mode = config.value.networking?.default_mode || ''; + return mode === 'bridge' || mode === 'macvtap'; + }); const defaultNetworkingLabel = computed(() => { const mode = config.value.networking?.default_mode || ''; if (mode === 'bridge') { @@ -389,6 +447,7 @@ fi host_address: port.host_address || '127.0.0.1', host_port: typeof port.host_port === 'number' ? port.host_port : null, vm_port: typeof port.vm_port === 'number' ? port.vm_port : null, + nic_index: typeof port.nic_index === 'number' ? port.nic_index : null, })); const normalizePorts = (ports: PortFormEntry[] = []): VmmTypes.IPortMapping[] => @@ -399,11 +458,23 @@ fi port.host_port === null || port.host_port === undefined ? Number.NaN : Number(port.host_port); const vmPort = port.vm_port === null || port.vm_port === undefined ? Number.NaN : Number(port.vm_port); + // An unpinned mapping must stay unpinned rather than become NIC 0: + // the VMM's own default is the first user-mode NIC, not the first NIC. + // `v-model.number` hands back the raw string when it does not parse, + // so a box the operator cleared arrives as `''`. `Number('')` is 0, + // which would pin to NIC 0 the mapping they just unpinned. + const nicIndex = + port.nic_index === null || port.nic_index === undefined || port.nic_index === '' + ? undefined + : Number(port.nic_index); return { protocol, host_address: (port.host_address || '127.0.0.1').trim() || '127.0.0.1', host_port: hostPort, vm_port: vmPort, + ...(Number.isInteger(nicIndex) && (nicIndex as number) >= 0 + ? { nic_index: nicIndex } + : {}), }; }) .filter( @@ -411,13 +482,7 @@ fi port.protocol.length > 0 && Number.isFinite(port.host_port) && Number.isFinite(port.vm_port), - ) - .map((port) => ({ - protocol: port.protocol, - host_address: port.host_address, - host_port: port.host_port, - vm_port: port.vm_port, - })); + ); const cloneNetworks = (configuration?: VmConfiguration | null): NetworkFormEntry[] => { const configured = configuration?.networks && configuration.networks.length > 0 @@ -426,16 +491,67 @@ fi return configured.map((network) => ({ mode: network.mode || '', bridge_name: network.bridge_name || '', + parent: network.parent || '', + vhost: network.vhost === null || network.vhost === undefined ? '' : (network.vhost ? 'on' : 'off'), + queues: network.queues ? String(network.queues) : '', })); }; + // Queue pairs, whatever shape the model is in. + // + // Not a string: Vue's `v-model` casts for ``, so this + // field is a `string` while it holds a value loaded from `GetInfo` and a + // `number` the moment the operator types into it. Assuming either one is how + // this threw a `TypeError` out of every deploy that set a queue count. + // + // The number input already refuses everything but a numeric literal, and the + // cast turns `2.7` into `2.7` rather than into `parseInt`'s `2`, so what is + // left to check is that the value is a whole number at least one. The node's + // cap is deliberately *not* checked here: the server widens it by whatever a + // VM already holds, so a client-side copy would refuse an update the server + // accepts and leave that VM's networking uneditable. + const parseQueueCount = (raw: unknown, label: string): number | undefined => { + if (raw === null || raw === undefined || raw === '') { + return undefined; + } + const queues = typeof raw === 'number' ? raw : Number(String(raw).trim()); + if (!Number.isInteger(queues) || queues < 1) { + throw new Error(`${label}: queue pairs must be a whole number of at least 1, or empty to follow the vCPU count; got '${raw}'`); + } + return queues; + }; + + // Nothing is filtered out. A mode-less entry is the "keep the node's backend, + // change only the data plane" override the RPC accepts, and is what a VM + // deployed that way reports back; dropping it would delete a NIC and renumber + // the ones after it, which changes their MAC addresses. + // A row added here starts with no mode, which the RPC reads as "keep the + // node's backend" rather than as a missing field, so the only way to reach an + // entry it refuses is to empty a loaded one's bridge or parent, and that + // earns an error rather than silence. const normalizeNetworks = (networks: NetworkFormEntry[] = []): VmmTypes.INetworkingConfig[] => networks - .map((network) => ({ - mode: (network.mode || '').trim(), - bridge_name: network.mode === 'bridge' ? (network.bridge_name || '').trim() : '', - })) - .filter((network) => network.mode.length > 0); + .map((network, index) => { + // Leave vhost and queues unset unless the operator picked something, so + // the node keeps owning them and can still change them later. + const entry: VmmTypes.INetworkingConfig = { + mode: (network.mode || '').trim(), + bridge_name: network.mode === 'bridge' ? (network.bridge_name || '').trim() : '', + parent: network.mode === 'macvtap' ? (network.parent || '').trim() : '', + }; + // The tuning controls are hidden for user mode, so sending values the + // operator cannot see would fail the deploy with nothing to fix. + if (network.mode !== 'user') { + if (network.vhost === 'on' || network.vhost === 'off') { + entry.vhost = network.vhost === 'on'; + } + const queues = parseQueueCount(network.queues, `network ${index + 1}`); + if (queues !== undefined) { + entry.queues = queues; + } + } + return entry; + }); function networkModeLabel(mode?: string | null) { if (!mode) { @@ -502,6 +618,7 @@ type CreateVmPayloadSource = { hugepages?: boolean; pin_numa?: boolean; no_tee?: boolean; + simulated_tee?: string; networks?: NetworkFormEntry[]; gpus?: VmmTypes.IGpuConfig; kms_urls?: string[]; @@ -526,6 +643,7 @@ type CreateVmPayloadSource = { hugepages: !!source.hugepages, pin_numa: !!source.pin_numa, no_tee: source.no_tee ?? false, + simulated_tee: source.simulated_tee || undefined, networks: normalizedNetworks, gpus: source.gpus, kms_urls: source.kms_urls?.filter((url) => url && url.trim().length) ?? [], @@ -718,18 +836,49 @@ type CreateVmPayloadSource = { return image?.version; }; - const verGE = (versionStr: string, otherVersionStr: string) => { - const versionParts = versionStr.split('.').map(Number); - const otherParts = otherVersionStr.split('.').map(Number); - return ( - versionParts[0] > otherParts[0] || - (versionParts[0] === otherParts[0] && versionParts[1] > otherParts[1]) || - (versionParts[0] === otherParts[0] && versionParts[1] === otherParts[1] && versionParts[2] >= otherParts[2]) - ); + // Mirror of dstack-types::version::Version::parse. The prerelease/build + // suffix is stripped at the first '-' or '+', then major.minor[.patch] are + // read as decimal integers with patch defaulting to 0. So 0.6.1, 0.6.1-rc1, + // 0.6.1.rc1 and 0.6.1+build.5 all parse to the same version: a release + // candidate is built from the code of the version it is a candidate for and + // must not be feature-gated down to the previous release. + // + // Returns null when the string is absent or not parseable; verGE then + // reports false rather than silently comparing against NaN. + const parseVer = (v: string | undefined): [number, number, number] | null => { + if (!v) { + return null; + } + // Deliberately not `Number()`: it accepts ' 6', '0x10' and '' where the + // Rust parser rejects them, which would let the two sides disagree. + const num = (s: string | undefined): number | null => + s !== undefined && /^\d+$/.test(s) ? Number(s) : null; + const parts = v.trim().split(/[-+]/)[0].split('.'); + const major = num(parts[0]); + const minor = num(parts[1]); + const patch = parts.length > 2 ? num(parts[2]) : 0; + if (major === null || minor === null || patch === null) { + return null; + } + return [major, minor, patch]; + }; + + const verGE = (versionStr: string | undefined, otherVersionStr: string) => { + const version = parseVer(versionStr); + const other = parseVer(otherVersionStr); + if (!version || !other) { + return false; + } + for (let i = 0; i < 3; i++) { + if (version[i] !== other[i]) { + return version[i] > other[i]; + } + } + return true; }; const appComposeManifestVersion = (versionStr: string | undefined): number | string => { - if (versionStr && verGE(versionStr, '0.6.0')) { + if (verGE(versionStr, '0.6.0')) { return '3'; } return 2; @@ -742,9 +891,6 @@ type CreateVmPayloadSource = { network_info: false, compose_version: 1, }; - if (!versionStr) { - return features; - } if (verGE(versionStr, '0.3.3')) { features.progress = true; features.graceful_shutdown = true; @@ -804,6 +950,9 @@ type CreateVmPayloadSource = { async function makeAppComposeFile() { const osVersion = imageVersion(vmForm.value.image); + if (nonEmptyInitScripts(vmForm.value.initScripts).length > 1 && !verGE(osVersion, '0.6.0')) { + throw new Error('Multiple init scripts require a dstack image version 0.6.0 or newer'); + } const appCompose: Record = { manifest_version: appComposeManifestVersion(osVersion), name: vmForm.value.name, @@ -813,10 +962,13 @@ type CreateVmPayloadSource = { public_logs: vmForm.value.public_logs, public_sysinfo: vmForm.value.public_sysinfo, public_tcbinfo: vmForm.value.public_tcbinfo, - key_provider_id: vmForm.value.key_provider_id, + key_provider_id: vmForm.value.key_provider === 'kms' || vmForm.value.key_provider === 'local' + ? vmForm.value.key_provider_id + : '', allowed_envs: vmForm.value.encryptedEnvs.map((env) => env.key), no_instance_id: !vmForm.value.gateway_enabled, secure_time: false, + storage_discard: vmForm.value.storage_discard, }; if (vmForm.value.key_provider !== undefined) { @@ -835,8 +987,9 @@ type CreateVmPayloadSource = { appCompose.storage_fs = vmForm.value.storage_fs; } - if (vmForm.value.initScript?.trim()) { - appCompose.init_script = vmForm.value.initScript; + const initScripts = compatibleInitScripts(vmForm.value.initScripts); + if (initScripts !== undefined) { + appCompose.init_script = initScripts; } if (vmForm.value.preLaunchScript?.trim()) { @@ -848,6 +1001,10 @@ type CreateVmPayloadSource = { appCompose.swap_size = swapBytes; } + if (vmForm.value.event_log_version && vmForm.value.event_log_version !== 1) { + appCompose.event_log_version = vmForm.value.event_log_version; + } + const launchToken = vmForm.value.encryptedEnvs.find((env) => env.key === 'APP_LAUNCH_TOKEN'); if (launchToken) { appCompose.launch_token_hash = await calcComposeHash(launchToken.value); @@ -868,6 +1025,9 @@ type CreateVmPayloadSource = { docker_compose_file: updateDialog.value.dockerComposeFile || currentAppCompose.docker_compose_file, }; const targetManifestVersion = appComposeManifestVersion(imageVersion(updateDialog.value.image)); + if (nonEmptyInitScripts(updateDialog.value.initScripts).length > 1 && targetManifestVersion !== '3') { + throw new Error('Multiple init scripts require a dstack image version 0.6.0 or newer'); + } if (targetManifestVersion === '3') { appCompose.manifest_version = targetManifestVersion; } @@ -880,7 +1040,7 @@ type CreateVmPayloadSource = { appCompose.launch_token_hash = await calcComposeHash(launchToken.value); } } - appCompose.init_script = updateDialog.value.initScript?.trim() || undefined; + appCompose.init_script = compatibleInitScripts(updateDialog.value.initScripts); appCompose.pre_launch_script = updateDialog.value.preLaunchScript?.trim() || undefined; const swapBytes = Math.max(0, Math.round(updateDialog.value.swap_size || 0)); @@ -947,6 +1107,11 @@ type CreateVmPayloadSource = { function showDeployDialog() { showCreateDialog.value = true; vmForm.value.encryptedEnvs = []; + // A cancelled deploy and "Clone config" both leave their networking behind + // in the shared form. Carrying it into the next deploy would silently pin + // that VM's backend, bridge, macvtap parent and data plane to another VM's + // -- invisibly, since the operator never opened the Networking section. + vmForm.value.networks = []; vmForm.value.app_id = null; vmForm.value.swapValue = 0; vmForm.value.swapUnit = 'GB'; @@ -969,7 +1134,7 @@ type CreateVmPayloadSource = { vm: detailedVm, updateCompose: false, dockerComposeFile: detailedVm.appCompose.docker_compose_file || '', - initScript: detailedVm.appCompose.init_script || '', + initScripts: initScriptList(detailedVm.appCompose.init_script), preLaunchScript: detailedVm.appCompose.pre_launch_script || '', encryptedEnvs: [], resetSecrets: false, @@ -1080,6 +1245,7 @@ type CreateVmPayloadSource = { hugepages: vmForm.value.hugepages, pin_numa: vmForm.value.pin_numa, no_tee: vmForm.value.no_tee, + simulated_tee: vmForm.value.simulated_tee, networks: vmForm.value.networks, gpus: configGpu(vmForm.value) || undefined, kms_urls: vmForm.value.kms_urls, @@ -1092,7 +1258,7 @@ type CreateVmPayloadSource = { loadVMList(); } catch (error) { recordError('Error creating VM', error); - alert('Failed to create VM'); + alert(error instanceof Error ? error.message : 'Failed to create VM'); } } @@ -1178,7 +1344,7 @@ type CreateVmPayloadSource = { loadVMList(); } catch (error) { recordError('error upgrading VM', error); - alert('failed to upgrade VM'); + alert(error instanceof Error ? error.message : 'Failed to upgrade VM'); } } @@ -1208,7 +1374,7 @@ type CreateVmPayloadSource = { name: `${config.name || vm.name}`, image: config.image || '', dockerComposeFile: theVm.appCompose?.docker_compose_file || '', - initScript: theVm.appCompose?.init_script || '', + initScripts: initScriptList(theVm.appCompose?.init_script), preLaunchScript: theVm.appCompose?.pre_launch_script || '', vcpu: config.vcpu || 1, memory: config.memory || 0, @@ -1223,6 +1389,7 @@ type CreateVmPayloadSource = { encryptedEnvs: [], // Clear environment variables ports: [], // Clear port mappings storage_fs: theVm.appCompose?.storage_fs || 'zfs', + storage_discard: theVm.appCompose?.storage_discard ?? true, app_id: config.app_id || '', kms_urls: config.kms_urls || [], key_provider: getKeyProvider(theVm), @@ -1236,8 +1403,10 @@ type CreateVmPayloadSource = { pin_numa: !!config.pin_numa, hugepages: !!config.hugepages, no_tee: !!config.no_tee, + simulated_tee: config.simulated_tee || '', user_config: config.user_config || '', stopped: !!config.stopped, + event_log_version: theVm.appCompose?.event_log_version || 1, }; // Show Create VM dialog instead of Clone Config dialog @@ -1265,6 +1434,7 @@ type CreateVmPayloadSource = { hugepages: source.hugepages, pin_numa: source.pin_numa, no_tee: source.no_tee, + simulated_tee: source.simulated_tee, networks: source.networks, gpus: source.gpus, kms_urls: source.kms_urls, @@ -1360,11 +1530,6 @@ type CreateVmPayloadSource = { window.open('/api-docs/docs', '_blank', 'noopener'); } - function openLegacyUi() { - closeSystemMenu(); - window.open('/v0', '_blank', 'noopener'); - } - function shortUptime(uptime?: string | null) { if (!uptime) { return '-'; @@ -1767,7 +1932,10 @@ type CreateVmPayloadSource = { config, networkingModes, defaultBridge, + maxNetQueues, defaultNetworkingLabel, + defaultModeTunable, + defaultVhostOn, composeHashPreview, updateComposeHashPreview, showDeployDialog, @@ -1807,7 +1975,6 @@ type CreateVmPayloadSource = { toggleSystemMenu, closeSystemMenu, openApiDocs, - openLegacyUi, reloadVMs, devMode, toggleDevMode, diff --git a/dstack/vmm/ui/src/styles/main.css b/dstack/vmm/ui/src/styles/main.css index 7a7b0ee72..113109a42 100644 --- a/dstack/vmm/ui/src/styles/main.css +++ b/dstack/vmm/ui/src/styles/main.css @@ -620,7 +620,7 @@ h1, h2, h3, h4, h5, h6 { .runtime-network-item { display: grid; - grid-template-columns: 1.1fr 0.6fr 0.8fr 1.4fr; + grid-template-columns: 1.1fr 0.6fr 0.8fr 1.4fr 0.7fr 0.7fr; gap: 12px; overflow-wrap: anywhere; } @@ -1420,13 +1420,35 @@ h1, h2, h3, h4, h5, h6 { font-size: 14px; } +/* Fixed track widths summed to a floor wider than the dialog, which put a + horizontal scrollbar under the Networking section on any window narrower than + about 930px. The tracks shrink and the tuning pair wraps instead. */ .network-config-row { display: grid; - grid-template-columns: 160px minmax(180px, 1fr) 100px; + grid-template-columns: minmax(120px, 160px) minmax(140px, 1fr) minmax(200px, auto) auto; gap: 12px; align-items: center; } +.network-config-tuning { + display: flex; + flex-wrap: wrap; + gap: 8px; + min-width: 0; +} + +.network-config-tuning select { + flex: 1 1 140px; + min-width: 110px; + max-width: 160px; +} + +.network-config-tuning input { + flex: 1 1 150px; + min-width: 90px; + max-width: 190px; +} + .network-config-placeholder { min-height: 1px; } diff --git a/dstack/vmm/ui/src/templates/app.html b/dstack/vmm/ui/src/templates/app.html index 7e38806c7..965f7bcc2 100644 --- a/dstack/vmm/ui/src/templates/app.html +++ b/dstack/vmm/ui/src/templates/app.html @@ -57,13 +57,6 @@

dstack-vmm

API Docs - - -
- {% if records %} - - - - - - - - - - - - - {% for record in records %} - - - - - - - - - {% endfor %} - -
Zone IDTypeNameContentCreatedActions
{{ record.zone_id[:12] }}...{{ record.type }}{{ record.name }}{{ record.content }}{{ record.created_on[:19] }} - -
- {% else %} -
-

No DNS records yet. Records created via API will appear here.

-
- {% endif %} -
- - -
-
-

Recent API Requests

- -
-
- {% if logs %} - {% for log in logs %} -
- {{ log.timestamp }} - {{ log.method }} - {{ log.path }} - - ({{ log.status_code }}) - - {% if log.request %} -
- Request/Response - 
Request: {{ log.request | tojson(indent=2) }}
- 
Response: {{ log.response | tojson(indent=2) }}
-
- {% endif %} -
- {% endfor %} - {% else %} -
-

No API requests yet.

-
- {% endif %} -
-
- -
-
-

API Usage

-
-
-

Base URL:

- 

-            

-        

-    
-
-    
-
-    
-
-
-"""
-
-
-@app.route("/")
-def management_ui():
-    """Render the management UI."""
-    all_records = []
-    for zone_id, records in dns_records.items():
-        all_records.extend(records.values())
-
-    # Sort by created time, newest first
-    all_records.sort(key=lambda r: r.get("created_on", ""), reverse=True)
-
-    return render_template_string(
-        MANAGEMENT_HTML,
-        zone_count=len(dns_records),
-        record_count=sum(len(r) for r in dns_records.values()),
-        request_count=len(request_logs),
-        records=all_records,
-        logs=request_logs[:20],
-        port=os.environ.get("PORT", 8080),
-    )
-
-
-# ==================== Management API ====================
-
-
-@app.route("/api/records", methods=["DELETE"])
-def clear_all_records():
-    """Clear all DNS records."""
-    dns_records.clear()
-    return jsonify({"success": True})
-
-
-@app.route("/api/records//", methods=["DELETE"])
-def delete_record_ui(zone_id, record_id):
-    """Delete a specific record from UI."""
-    if zone_id in dns_records and record_id in dns_records[zone_id]:
-        del dns_records[zone_id][record_id]
-    return jsonify({"success": True})
-
-
-@app.route("/api/logs", methods=["DELETE"])
-def clear_logs():
-    """Clear request logs."""
-    request_logs.clear()
-    return jsonify({"success": True})
-
-
-@app.route("/api/records", methods=["GET"])
-def get_all_records():
-    """Get all records as JSON."""
-    all_records = []
-    for zone_id, records in dns_records.items():
-        all_records.extend(records.values())
-    return jsonify(all_records)
-
-
-@app.route("/health")
-def health():
-    """Health check endpoint."""
-    return jsonify(
-        {"status": "healthy", "records": sum(len(r) for r in dns_records.values())}
-    )
-
-
-if __name__ == "__main__":
-    port = int(os.environ.get("PORT", 8080))
-    debug = os.environ.get("DEBUG", "false").lower() == "true"
-
-    print(f"""
-    ╔═══════════════════════════════════════════════════════════════╗
-    ║         Mock Cloudflare DNS API Server                        ║
-    ╠═══════════════════════════════════════════════════════════════╣
-    ║  Management UI:  http://localhost:{port}/                         ║
-    ║  API Base URL:   http://localhost:{port}/client/v4                ║
-    ╚═══════════════════════════════════════════════════════════════╝
-    """)
-
-    app.run(host="0.0.0.0", port=port, debug=debug)
diff --git a/tools/mock-cf-dns-api/docker-compose.yml b/tools/mock-cf-dns-api/docker-compose.yml
deleted file mode 100644
index 951bd68b7..000000000
--- a/tools/mock-cf-dns-api/docker-compose.yml
+++ /dev/null
@@ -1,17 +0,0 @@
-# SPDX-FileCopyrightText: 2024-2025 Phala Network 
-#
-# SPDX-License-Identifier: Apache-2.0
-
-version: "3.8"
-
-services:
-  mock-cf-dns-api:
-    image: kvin/mock-cf-dns-api:latest
-    ports:
-      - "8080:8080"
-    environment:
-      - PORT=8080
-      - DEBUG=false
-      # Optional: comma-separated list of valid API tokens
-      # - CF_API_TOKENS=token1,token2
-    restart: unless-stopped
diff --git a/tools/mock-cf-dns-api/requirements.txt b/tools/mock-cf-dns-api/requirements.txt
deleted file mode 100644
index a36229c7e..000000000
--- a/tools/mock-cf-dns-api/requirements.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-flask>=3.0.0
-gunicorn>=21.0.0
diff --git a/tools/mock-cf-dns/Dockerfile b/tools/mock-cf-dns/Dockerfile
new file mode 100644
index 000000000..679dc4164
--- /dev/null
+++ b/tools/mock-cf-dns/Dockerfile
@@ -0,0 +1,9 @@
+# SPDX-FileCopyrightText: © 2026 Phala Network 
+#
+# SPDX-License-Identifier: Apache-2.0
+
+FROM python:3.12-slim
+WORKDIR /app
+COPY server.py /app/server.py
+EXPOSE 8080/tcp 53/udp 53/tcp
+CMD ["python", "/app/server.py"]
diff --git a/tools/mock-cf-dns/server.py b/tools/mock-cf-dns/server.py
new file mode 100755
index 000000000..b1e1f5f55
--- /dev/null
+++ b/tools/mock-cf-dns/server.py
@@ -0,0 +1,535 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: © 2026 Phala Network 
+# SPDX-License-Identifier: Apache-2.0
+
+"""Tiny Cloudflare DNS API + UDP TXT DNS mock for dstack gateway E2E.
+
+It implements just the Cloudflare endpoints used by certbot's CloudflareClient:
+  GET    /client/v4/zones
+  GET    /client/v4/zones//dns_records?name=
+  POST   /client/v4/zones//dns_records
+  DELETE /client/v4/zones//dns_records/
+
+It tracks one thing beyond storing records: names whose issuer CAA set was
+emptied, served at /api/caa-gaps, because replacing CAA is meant to leave one
+record standing throughout and a gap is invisible once the replacement is done.
+
+It also serves TXT answers on 53, over both UDP and TCP, so Pebble can validate
+DNS-01 and dns-persist-01 for real rather than being run with
+PEBBLE_VA_ALWAYS_VALID=1. TCP is not optional: Pebble sets its DNS client to
+`Net = "tcp"` whenever it is given `-dnsserver`.
+
+Questions outside the configured zones are forwarded upstream, so this can be a
+client's only resolver rather than only a CA's `-dnsserver`: certbot resolves
+its own challenge records through it while still reaching the other containers
+by name. Every question is logged and served at /api/dns-queries, so a test can
+assert that a name was actually looked up.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import socket
+import struct
+import threading
+import time
+import urllib.parse
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import Any
+
+STATE_LOCK = threading.RLock()
+RECORDS: list[dict[str, Any]] = []
+# Every DNS question this mock was asked, so a test can assert that the client
+# under test resolved a name rather than inferring it from what happened next.
+QUERIES: list[dict[str, Any]] = []
+MAX_QUERIES = 2000
+QTYPE_ANY = 255
+NEXT_ID = 1
+# Names whose issuer CAA set went from non-empty to empty at some point.
+#
+# A client replacing those records is meant to keep at least one in place
+# throughout: absent CAA is not "no issuer may" but "any issuer may", so a gap
+# is a window in which any CA could have issued. It closes as soon as the
+# replacement finishes and is invisible in the result, so it is recorded as it
+# happens.
+CAA_GAPS: set[str] = set()
+
+
+def _zones() -> list[dict[str, str]]:
+    raw = os.environ.get("MOCK_CF_ZONES", "e2e.test test local")
+    names = [
+        z.strip().strip(".").lower() for z in re.split(r"[,\s]+", raw) if z.strip()
+    ]
+    if not names:
+        names = ["e2e.test"]
+    out = []
+    seen = set()
+    for name in names:
+        if name in seen:
+            continue
+        seen.add(name)
+        out.append({"id": zone_id_for(name), "name": name})
+    return out
+
+
+def zone_id_for(name: str) -> str:
+    """Return a deterministic mock zone ID for a DNS name."""
+    safe = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") or "zone"
+    return f"zone-{safe}"
+
+
+def _json(handler: BaseHTTPRequestHandler, code: int, body: Any) -> None:
+    data = json.dumps(body, sort_keys=True).encode()
+    handler.send_response(code)
+    handler.send_header("content-type", "application/json")
+    handler.send_header("content-length", str(len(data)))
+    handler.end_headers()
+    handler.wfile.write(data)
+
+
+def _issuer_caa_count(name: str) -> int:
+    """How many issue/issuewild CAA records currently sit at `name`."""
+    wanted = name.strip(".").lower()
+    return sum(
+        1
+        for r in RECORDS
+        if r["type"] == "CAA"
+        and r["name"].strip(".").lower() == wanted
+        and (r["content"].split() + ["", ""])[1] in ("issue", "issuewild")
+    )
+
+
+def _note_caa_gap(name: str, before: int) -> None:
+    """Record `name` if its issuer CAA set just emptied out.
+
+    Called with STATE_LOCK held, `before` sampled ahead of the mutation.
+    """
+    if before > 0 and _issuer_caa_count(name) == 0:
+        CAA_GAPS.add(name.strip(".").lower())
+
+
+def _record_content(payload: dict[str, Any]) -> str:
+    if "content" in payload:
+        return str(payload.get("content") or "")
+    data = payload.get("data") or {}
+    if payload.get("type") == "CAA" and isinstance(data, dict):
+        return f'{data.get("flags", 0)} {data.get("tag", "issue")} "{data.get("value", "")}"'
+    return ""
+
+
+def _authorized(handler: BaseHTTPRequestHandler) -> bool:
+    """Require the configured Cloudflare bearer token for API operations."""
+    expected = os.environ.get("MOCK_CF_API_TOKEN", "test-token")
+    if handler.headers.get("Authorization", "") == f"Bearer {expected}":
+        return True
+    _json(
+        handler,
+        403,
+        {"success": False, "errors": [{"message": "invalid API token"}]},
+    )
+    return False
+
+
+class Handler(BaseHTTPRequestHandler):
+    """Handle the mock Cloudflare HTTP API."""
+
+    server_version = "dstack-mock-cf-dns/0.1"
+
+    def log_message(self, fmt: str, *args: Any) -> None:
+        """Emit request logs when debug logging is enabled."""
+        if os.environ.get("DEBUG", "").lower() in {"1", "true", "yes"}:
+            super().log_message(fmt, *args)
+
+    def do_GET(self) -> None:  # noqa: N802
+        """Handle supported HTTP GET endpoints."""
+        parsed = urllib.parse.urlparse(self.path)
+        path = parsed.path.rstrip("/") or "/"
+        query = urllib.parse.parse_qs(parsed.query)
+        if path == "/health":
+            _json(self, 200, {"ok": True})
+            return
+        if path == "/api/caa-gaps":
+            with STATE_LOCK:
+                _json(self, 200, {"names": sorted(CAA_GAPS)})
+            return
+        if path == "/api/records":
+            with STATE_LOCK:
+                _json(self, 200, {"records": RECORDS})
+            return
+        if path == "/api/dns-queries":
+            with STATE_LOCK:
+                _json(self, 200, {"queries": QUERIES})
+            return
+        if path.startswith("/client/v4/") and not _authorized(self):
+            return
+        if path == "/client/v4/zones":
+            zones = _zones()
+            _json(
+                self,
+                200,
+                {
+                    "success": True,
+                    "result": zones,
+                    "result_info": {
+                        "page": 1,
+                        "per_page": 50,
+                        "total_pages": 1,
+                        "count": len(zones),
+                        "total_count": len(zones),
+                    },
+                },
+            )
+            return
+        m = re.fullmatch(r"/client/v4/zones/([^/]+)/dns_records", path)
+        if m:
+            name = (query.get("name") or [""])[0].strip().strip(".").lower()
+            with STATE_LOCK:
+                records = [
+                    r
+                    for r in RECORDS
+                    if not name or r["name"].strip(".").lower() == name
+                ]
+            _json(
+                self,
+                200,
+                {
+                    "success": True,
+                    "result": records,
+                    "result_info": {"page": 1, "per_page": 100, "total_pages": 1},
+                },
+            )
+            return
+        _json(
+            self, 404, {"success": False, "errors": [{"message": f"not found: {path}"}]}
+        )
+
+    def do_POST(self) -> None:  # noqa: N802
+        """Create a mock DNS record."""
+        global NEXT_ID
+        parsed = urllib.parse.urlparse(self.path)
+        path = parsed.path.rstrip("/") or "/"
+        m = re.fullmatch(r"/client/v4/zones/([^/]+)/dns_records", path)
+        if not m:
+            _json(
+                self,
+                404,
+                {"success": False, "errors": [{"message": f"not found: {path}"}]},
+            )
+            return
+        if not _authorized(self):
+            return
+        length = int(self.headers.get("content-length", "0") or "0")
+        payload = json.loads(self.rfile.read(length) or b"{}")
+        with STATE_LOCK:
+            record_id = f"rec-{NEXT_ID}"
+            NEXT_ID += 1
+            record = {
+                "id": record_id,
+                "zone_id": m.group(1),
+                "type": str(payload.get("type") or "TXT").upper(),
+                "name": str(payload.get("name") or "").strip().strip("."),
+                "content": _record_content(payload),
+                "ttl": int(payload.get("ttl") or 60),
+                "created_on": int(time.time()),
+            }
+            RECORDS.append(record)
+        _json(self, 200, {"success": True, "result": record})
+
+    def do_DELETE(self) -> None:  # noqa: N802
+        """Delete a mock DNS record, or forget the CAA gaps seen so far."""
+        parsed = urllib.parse.urlparse(self.path)
+        path = parsed.path.rstrip("/") or "/"
+        if path == "/api/caa-gaps":
+            with STATE_LOCK:
+                CAA_GAPS.clear()
+            _json(self, 200, {"success": True})
+            return
+        m = re.fullmatch(r"/client/v4/zones/([^/]+)/dns_records/([^/]+)", path)
+        if not m:
+            _json(
+                self,
+                404,
+                {"success": False, "errors": [{"message": f"not found: {path}"}]},
+            )
+            return
+        if not _authorized(self):
+            return
+        record_id = urllib.parse.unquote(m.group(2))
+        with STATE_LOCK:
+            doomed = next((r for r in RECORDS if r["id"] == record_id), None)
+            caa_name = doomed["name"] if doomed and doomed["type"] == "CAA" else None
+            caa_before = _issuer_caa_count(caa_name) if caa_name else 0
+            before = len(RECORDS)
+            RECORDS[:] = [r for r in RECORDS if r["id"] != record_id]
+            removed = before != len(RECORDS)
+            if caa_name:
+                _note_caa_gap(caa_name, caa_before)
+        _json(
+            self,
+            200,
+            {"success": True, "result": {"id": record_id, "removed": removed}},
+        )
+
+
+def parse_qname(packet: bytes, offset: int = 12) -> tuple[str, int]:
+    """Parse a DNS question name and return its value and end offset."""
+    labels: list[str] = []
+    while True:
+        if offset >= len(packet):
+            raise ValueError("bad qname")
+        length = packet[offset]
+        offset += 1
+        if length == 0:
+            break
+        labels.append(packet[offset : offset + length].decode("ascii", "ignore"))
+        offset += length
+    return ".".join(labels).strip(".").lower(), offset
+
+
+def txt_rdata(text: str) -> bytes:
+    """Encode text as DNS TXT record data."""
+    raw = text.encode()
+    chunks = [raw[i : i + 255] for i in range(0, len(raw), 255)] or [b""]
+    return b"".join(bytes([len(c)]) + c for c in chunks)
+
+
+def wire_name(name: str) -> bytes:
+    """Encode a domain name uncompressed, for use in RDATA."""
+    out = b""
+    for label in name.strip(".").split("."):
+        raw = label.encode("ascii", "ignore")[:63]
+        out += bytes([len(raw)]) + raw
+    return out + b"\x00"
+
+
+def ns_address() -> str | None:
+    """Address to advertise as this mock's nameserver, if it is to advertise one.
+
+    A client that reads a challenge record straight from the authoritative
+    servers -- which is the only way to dodge a recursive resolver's negative
+    cache -- gets there by asking NS for the zone and then A for each answer.
+    Unset, this mock answers neither and that client falls back to its system
+    resolver, so the authoritative path never runs. Set to the address this
+    container is reachable at, and the path is exercised end to end.
+    """
+    return os.environ.get("MOCK_CF_NS_ADDR") or None
+
+
+def _synthesized(name: str, qtype: int) -> list[tuple[int, bytes]]:
+    """NS at a zone apex and A for the nameserver it names.
+
+    Only at the apex: a name below it must answer NODATA, or a client walking
+    up from `_acme-challenge.` looking for the zone cut would stop at the
+    wrong place -- and walking up is what it does against a real zone.
+    """
+    addr = ns_address()
+    if not addr:
+        return []
+    apexes = {zone["name"] for zone in _zones()}
+    if qtype in (2, QTYPE_ANY) and name in apexes:
+        return [(2, wire_name(f"ns.{name}"))]
+    if qtype in (1, QTYPE_ANY) and name.startswith("ns.") and name[3:] in apexes:
+        return [(1, socket.inet_aton(addr))]
+    return []
+
+
+def _upstream() -> str:
+    """Where to send questions this mock is not authoritative for.
+
+    Docker's embedded resolver, which is what this container's own
+    `/etc/resolv.conf` points at, so service names keep resolving for whoever
+    is pointed here.
+    """
+    return os.environ.get("MOCK_DNS_UPSTREAM", "127.0.0.11")
+
+
+def _is_ours(name: str) -> bool:
+    """Whether `name` falls inside one of the configured mock zones."""
+    return any(
+        name == zone["name"] or name.endswith("." + zone["name"]) for zone in _zones()
+    )
+
+
+def _forward(packet: bytes) -> bytes:
+    """Ask the upstream resolver and hand back its answer verbatim.
+
+    Without this the mock is only usable as a CA's `-dnsserver`, because a name
+    it does not know is answered NOERROR with no records -- which a resolver
+    reads as an authoritative "no such record" and does not retry elsewhere.
+    A client configured to use this as its only resolver would then fail to
+    resolve the other containers.
+    """
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    sock.settimeout(5)
+    try:
+        sock.sendto(packet, (_upstream(), 53))
+        return sock.recvfrom(65535)[0]
+    finally:
+        sock.close()
+
+
+def _record_query(name: str, qtype: int, answers: int, forwarded: bool) -> None:
+    with STATE_LOCK:
+        QUERIES.append(
+            {
+                "name": name,
+                "type": qtype,
+                "answers": answers,
+                "forwarded": forwarded,
+                "at": int(time.time()),
+            }
+        )
+        del QUERIES[:-MAX_QUERIES]
+
+
+def dns_response(packet: bytes) -> bytes:
+    """Build a DNS response containing matching TXT records."""
+    if len(packet) < 12:
+        return b""
+    txid = packet[:2]
+    qdcount = struct.unpack("!H", packet[4:6])[0]
+    if qdcount < 1:
+        return b""
+    name, qend = parse_qname(packet)
+    question = packet[12 : qend + 4]
+    qtype = (
+        struct.unpack("!H", packet[qend : qend + 2])[0]
+        if qend + 4 <= len(packet)
+        else 16
+    )
+    # Only answer for the zones this mock owns; anything else is the caller's
+    # ordinary name resolution and belongs upstream.
+    if not _is_ours(name):
+        try:
+            reply = _forward(packet)
+            _record_query(name, qtype, -1, True)
+            return reply
+        except Exception as exc:  # pragma: no cover - diagnostic only
+            _debug(f"forwarding {name} failed: {exc}")
+            _record_query(name, qtype, 0, True)
+            return (
+                txid + struct.pack("!HHHHH", 0x8182, 1, 0, 0, 0) + packet[12 : qend + 4]
+            )
+
+    with STATE_LOCK:
+        stored = [
+            r
+            for r in RECORDS
+            if r["type"] == "TXT" and r["name"].strip(".").lower() == name
+        ]
+    if qtype not in (16, QTYPE_ANY):
+        stored = []
+    answers = [(16, txt_rdata(r["content"])) for r in stored]
+    answers += _synthesized(name, qtype)
+
+    _record_query(name, qtype, len(answers), False)
+    header = txid + struct.pack("!HHHHH", 0x8180, 1, len(answers), 0, 0)
+    body = question
+    for rtype, rdata in answers:
+        body += b"\xc0\x0c" + struct.pack("!HHIH", rtype, 1, 60, len(rdata)) + rdata
+    return header + body
+
+
+def bind_host() -> str:
+    """Address the mock listens on, for both the HTTP API and DNS.
+
+    Defaults to every interface. This runs as a container on a private compose
+    network and its peers -- certbot writing records, and the ACME server
+    resolving them -- reach it by an address the process cannot know in
+    advance, so narrowing the default would answer nobody. Set MOCK_CF_BIND to
+    something specific when running it outside a container, where the default
+    would expose a test double that trusts every caller.
+    """
+    return os.environ.get("MOCK_CF_BIND", "0.0.0.0")
+
+
+def _debug(message: str) -> None:
+    if os.environ.get("DEBUG", "").lower() in {"1", "true", "yes"}:
+        print(message, flush=True)
+
+
+def dns_loop() -> None:
+    """Serve mock DNS responses over UDP."""
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    sock.bind((bind_host(), 53))
+    while True:
+        packet, addr = sock.recvfrom(4096)
+        try:
+            response = dns_response(packet)
+            if response:
+                sock.sendto(response, addr)
+        except Exception as exc:  # pragma: no cover - diagnostic only
+            _debug(f"dns error from {addr}: {exc}")
+
+
+def _serve_dns_over_tcp(conn: socket.socket, addr: Any) -> None:
+    """Answer one TCP DNS client, which frames each message with its length."""
+    try:
+        conn.settimeout(5)
+        while True:
+            head = _recv_exactly(conn, 2)
+            if head is None:
+                return
+            (length,) = struct.unpack("!H", head)
+            query = _recv_exactly(conn, length)
+            if query is None:
+                return
+            response = dns_response(query)
+            if not response:
+                return
+            conn.sendall(struct.pack("!H", len(response)) + response)
+    except Exception as exc:  # pragma: no cover - diagnostic only
+        _debug(f"dns/tcp error from {addr}: {exc}")
+    finally:
+        conn.close()
+
+
+def _recv_exactly(conn: socket.socket, count: int) -> bytes | None:
+    """Read exactly `count` bytes, or None if the peer stopped sending."""
+    buf = b""
+    while len(buf) < count:
+        chunk = conn.recv(count - len(buf))
+        if not chunk:
+            return None
+        buf += chunk
+    return buf
+
+
+def dns_tcp_loop() -> None:
+    """Serve mock DNS responses over TCP.
+
+    Pebble sets `dnsClient.Net = "tcp"` whenever it is given a custom resolver
+    (`-dnsserver`), so a UDP-only mock answers none of its queries and every
+    challenge has to be waved through with PEBBLE_VA_ALWAYS_VALID. RFC 1035
+    §4.2.2 length-prefixes each message on TCP; that framing is the only
+    difference from the UDP path.
+    """
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    sock.bind((bind_host(), 53))
+    sock.listen(16)
+    while True:
+        conn, addr = sock.accept()
+        threading.Thread(
+            target=_serve_dns_over_tcp, args=(conn, addr), daemon=True
+        ).start()
+
+
+def main() -> None:
+    """Run the mock Cloudflare API and DNS servers."""
+    threading.Thread(target=dns_loop, daemon=True).start()
+    threading.Thread(target=dns_tcp_loop, daemon=True).start()
+    port = int(os.environ.get("PORT", "8080"))
+    host = bind_host()
+    print(
+        f"mock CF API on {host}:{port}; DNS on {host}:53 (udp+tcp); zones={_zones()}",
+        flush=True,
+    )
+    ThreadingHTTPServer((host, port), Handler).serve_forever()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/mock-github-releases.py b/tools/mock-github-releases.py
new file mode 100755
index 000000000..5132507eb
--- /dev/null
+++ b/tools/mock-github-releases.py
@@ -0,0 +1,248 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: © 2026 Phala Network 
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Local GitHub Releases API overlay for dstackup development.
+
+Locally published releases shadow GitHub. Requests not satisfied locally are
+relayed to https://api.github.com. Release assets can be copied into --asset-dir
+by specifying an asset's "local_path" in the publish request.
+"""
+
+import argparse
+import hashlib
+import json
+import mimetypes
+import shutil
+import sys
+import urllib.error
+import urllib.request
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from urllib.parse import unquote, urlsplit
+
+UPSTREAM = "https://api.github.com"
+
+
+class State:
+    """Thread-safe-by-snapshot in-memory release overlay."""
+
+    def __init__(self, asset_dir: Path):
+        """Create an empty overlay and its local asset directory."""
+        self.asset_dir = asset_dir.resolve()
+        self.asset_dir.mkdir(parents=True, exist_ok=True)
+        self.releases = {}  # repo -> tag -> release; insertion order is newest first
+
+    def publish(self, repo, release, public_base):
+        """Publish a release as the newest local release for a repository."""
+        tag = release.get("tag_name")
+        if not isinstance(tag, str) or not tag:
+            raise ValueError("tag_name must be a non-empty string")
+        assets = []
+        for source in release.get("assets", []):
+            asset = dict(source)
+            local_path = asset.pop("local_path", None)
+            if local_path:
+                src = Path(local_path).resolve(strict=True)
+                name = asset.get("name") or src.name
+                if Path(name).name != name:
+                    raise ValueError(f"unsafe asset name: {name!r}")
+                dst = self.asset_dir / name
+                shutil.copyfile(src, dst)
+                digest = hashlib.sha256(dst.read_bytes()).hexdigest()
+                asset.update(
+                    name=name,
+                    browser_download_url=f"{public_base}/__assets/{name}",
+                    digest=f"sha256:{digest}",
+                )
+            if not asset.get("name") or not asset.get("browser_download_url"):
+                raise ValueError(
+                    "each asset needs name and browser_download_url, or local_path"
+                )
+            assets.append(asset)
+        release = dict(release, assets=assets)
+        # Build a new snapshot rather than mutating one observed by a concurrent
+        # GET handler. Assignment is atomic under CPython's GIL.
+        releases = self.releases.get(repo, {})
+        self.releases[repo] = {
+            tag: release,
+            **{old_tag: old for old_tag, old in releases.items() if old_tag != tag},
+        }
+        return release
+
+
+class Handler(BaseHTTPRequestHandler):
+    """Serve the local release overlay and relay misses to GitHub."""
+
+    server_version = "dstack-release-mock/1"
+
+    def json_response(self, status, value):
+        """Send a JSON response with a fixed content length."""
+        body = json.dumps(value, indent=2).encode()
+        self.send_response(status)
+        self.send_header("content-type", "application/json")
+        self.send_header("content-length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def do_POST(self):
+        """Publish a release through the local administration endpoint."""
+        # POST /__admin/repos/{owner}/{repo}/releases
+        parts = self.path.strip("/").split("/")
+        if (
+            len(parts) != 5
+            or parts[:2] != ["__admin", "repos"]
+            or parts[4:] != ["releases"]
+        ):
+            self.send_error(404)
+            return
+        try:
+            size = int(self.headers.get("content-length", "0"))
+            release = json.loads(self.rfile.read(size))
+            repo = "/".join(parts[2:4])
+            host = self.headers.get("host", f"127.0.0.1:{self.server.server_port}")
+            public_base = f"http://{host}"
+            release = self.server.state.publish(repo, release, public_base)
+            self.json_response(201, release)
+        except (ValueError, OSError, json.JSONDecodeError) as error:
+            self.json_response(400, {"error": str(error)})
+
+    def do_GET(self):
+        """Serve an asset or a GitHub-compatible releases API request."""
+        path = urlsplit(self.path).path
+        if path.startswith("/__assets/"):
+            self.serve_asset(unquote(path.removeprefix("/__assets/")))
+            return
+        # GitHub-compatible paths below /repos/{owner}/{repo}/releases...
+        parts = path.strip("/").split("/")
+        if len(parts) >= 4 and parts[0] == "repos":
+            repo = "/".join(parts[1:3])
+            suffix = parts[3:]
+            local = self.server.state.releases.get(repo, {})
+            if suffix == ["releases"] and local:
+                # Preserve local newest-first ordering, then add non-shadowed upstream releases.
+                upstream = self.fetch_upstream_json()
+                if isinstance(upstream, list):
+                    local_tags = set(local)
+                    upstream = [
+                        r for r in upstream if r.get("tag_name") not in local_tags
+                    ]
+                else:
+                    upstream = []
+                self.json_response(200, list(local.values()) + upstream)
+                return
+            if suffix == ["releases", "latest"] and local:
+                self.json_response(200, next(iter(local.values())))
+                return
+            if len(suffix) == 3 and suffix[:2] == ["releases", "tags"]:
+                release = local.get(unquote(suffix[2]))
+                if release:
+                    self.json_response(200, release)
+                    return
+        self.relay()
+
+    def serve_asset(self, name):
+        """Serve one basename-only file from the configured asset directory."""
+        if not name or Path(name).name != name:
+            self.send_error(400, "invalid asset name")
+            return
+        path = self.server.state.asset_dir / name
+        if not path.is_file():
+            self.send_error(404)
+            return
+        self.send_response(200)
+        self.send_header(
+            "content-type", mimetypes.guess_type(name)[0] or "application/octet-stream"
+        )
+        self.send_header("content-length", str(path.stat().st_size))
+        self.end_headers()
+        with path.open("rb") as source:
+            shutil.copyfileobj(source, self.wfile)
+
+    def upstream_request(self):
+        """Build the corresponding authenticated GitHub API request."""
+        url = UPSTREAM + self.path
+        headers = {
+            "user-agent": "dstack-release-mock",
+            "accept": "application/vnd.github+json",
+        }
+        auth = self.headers.get("authorization")
+        if auth:
+            headers["authorization"] = auth
+        return urllib.request.Request(url, headers=headers)
+
+    def fetch_upstream_json(self):
+        """Fetch upstream JSON for merging, returning an empty list on failure."""
+        try:
+            with urllib.request.urlopen(
+                self.upstream_request(), timeout=30
+            ) as response:
+                return json.load(response)
+        except (urllib.error.URLError, json.JSONDecodeError):
+            return []
+
+    def relay(self):
+        """Relay the current request and its HTTP status to GitHub."""
+        try:
+            with urllib.request.urlopen(
+                self.upstream_request(), timeout=30
+            ) as response:
+                body = response.read()
+                self.send_response(response.status)
+                self.send_header(
+                    "content-type",
+                    response.headers.get("content-type", "application/json"),
+                )
+                self.send_header("content-length", str(len(body)))
+                self.end_headers()
+                self.wfile.write(body)
+        except urllib.error.HTTPError as error:
+            body = error.read()
+            self.send_response(error.code)
+            self.send_header(
+                "content-type", error.headers.get("content-type", "application/json")
+            )
+            self.send_header("content-length", str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+        except urllib.error.URLError as error:
+            self.json_response(502, {"error": f"upstream request failed: {error}"})
+
+    def log_message(self, fmt, *args):
+        """Write HTTP access logs to stderr."""
+        sys.stderr.write("%s - %s\n" % (self.address_string(), fmt % args))
+
+
+class Server(ThreadingHTTPServer):
+    """Threading HTTP server carrying shared overlay state."""
+
+    def __init__(self, address, state):
+        """Bind the server and attach its overlay state."""
+        super().__init__(address, Handler)
+        self.state = state
+
+
+def main():
+    """Run the local releases API overlay until interrupted."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--listen", default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=8000)
+    parser.add_argument(
+        "--asset-dir", type=Path, default=Path("/tmp/dstack-release-mock-assets")
+    )
+    args = parser.parse_args()
+    server = Server((args.listen, args.port), State(args.asset_dir))
+    print(f"release API: http://{args.listen}:{args.port}/repos", flush=True)
+    print(
+        f"publish API: http://{args.listen}:{args.port}/__admin/repos/OWNER/REPO/releases",
+        flush=True,
+    )
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        pass
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/vm-runner/vm-runner.py b/tools/vm-runner/vm-runner.py
index 7bce7fe4f..0c8f1ac44 100755
--- a/tools/vm-runner/vm-runner.py
+++ b/tools/vm-runner/vm-runner.py
@@ -130,7 +130,29 @@ def update_guest_config(config_file: str, data: Dict):
         json.dump(config, f, indent=4)
 
 
-def gen_vm_config(vm_dir, host_port, manifest=None, os_image_hash=None):
+def detect_qemu_version(qemu_path):
+    """Return QEMU's `major.minor.micro`, or None if it cannot be determined.
+
+    MRTD depends on the QEMU version -- 8.x adds pages in two passes, 9.0+ in
+    one -- so a vm_config without this field makes the verifier fall back to a
+    default version and predict the wrong measurement.
+    """
+    try:
+        output = subprocess.check_output(
+            [qemu_path, '--version'], text=True, stderr=subprocess.DEVNULL)
+    except (OSError, subprocess.SubprocessError) as exc:
+        logging.warning("failed to run %s --version: %s", qemu_path, exc)
+        return None
+    match = re.search(r'QEMU emulator version (\d+\.\d+\.\d+)', output)
+    if not match:
+        logging.warning("could not parse QEMU version from: %s",
+                        output.splitlines()[0] if output else '')
+        return None
+    return match.group(1)
+
+
+def gen_vm_config(vm_dir, host_port, manifest=None, os_image_hash=None,
+                  qemu_version=None):
     shared_dir = os.path.join(vm_dir, 'shared')
     for filename in ['config.json', '.sys-config.json']:
         config_file = os.path.join(shared_dir, filename)
@@ -139,12 +161,15 @@ def gen_vm_config(vm_dir, host_port, manifest=None, os_image_hash=None):
             "host_vsock_port": host_port
         })
         if manifest:
+            vm_config = {
+                "os_image_hash": os_image_hash,
+                "cpu_count": manifest['vcpu'],
+                "memory_size": manifest['memory'] * 1024 * 1024
+            }
+            if qemu_version:
+                vm_config["qemu_version"] = qemu_version
             update_guest_config(config_file, {
-                "vm_config": json.dumps({
-                    "os_image_hash": os_image_hash,
-                    "cpu_count": manifest['vcpu'],
-                    "memory_size": manifest['memory'] * 1024 * 1024
-                })
+                "vm_config": json.dumps(vm_config)
             })
 
 
@@ -408,7 +433,8 @@ def run_instance(self, vm_dir: str, host_port: int, imgdir: Optional[str] = None
 
         os_image_hash = open(os.path.join(
             image_path, 'digest.txt'), 'r').read().strip()
-        gen_vm_config(vm_dir, host_port, manifest, os_image_hash)
+        gen_vm_config(vm_dir, host_port, manifest, os_image_hash,
+                      detect_qemu_version(self.config.qemu_path))
 
         mem_gb = manifest['memory'] // 1024
         vcpu_count = manifest['vcpu']
@@ -427,7 +453,10 @@ def run_instance(self, vm_dir: str, host_port: int, imgdir: Optional[str] = None
         # Prepare QEMU command
         cmd_args = []
         rootfs_image = os.path.join(image_path, img_metadata['rootfs'])
-        if rootfs_image.endswith('.img.verity'):
+        # Current images ship `rootfs.img.parted.verity`; older ones shipped
+        # `rootfs.img.verity`. Both are raw verity disks the initramfs finds by
+        # PARTLABEL, so attach either the same way.
+        if rootfs_image.endswith('.verity'):
             cmd_args.extend([
                 '-drive', f'file={rootfs_image},if=none,id=virtio-disk0,format=raw',
                 '-device', 'virtio-blk-pci,drive=virtio-disk0',