diff --git a/.github/workflows/docker-build-check.yml b/.github/workflows/docker-build-check.yml index 318df35a8..872385d81 100644 --- a/.github/workflows/docker-build-check.yml +++ b/.github/workflows/docker-build-check.yml @@ -11,7 +11,7 @@ name: Docker Build Check # does. on: push: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] paths: - 'dstack/**' - 'rust-toolchain.toml' @@ -19,8 +19,10 @@ on: - '.github/workflows/kms-release.yml' - '.github/workflows/gateway-release.yml' - '.github/workflows/verifier-release.yml' + - '.github/workflows/local-key-provider-release.yml' + - '.github/workflows/mkosi-build.yml' pull_request: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] paths: - 'dstack/**' - 'rust-toolchain.toml' @@ -28,6 +30,8 @@ on: - '.github/workflows/kms-release.yml' - '.github/workflows/gateway-release.yml' - '.github/workflows/verifier-release.yml' + - '.github/workflows/local-key-provider-release.yml' + - '.github/workflows/mkosi-build.yml' env: DSTACK_REV: ${{ github.event.pull_request.head.sha || github.sha }} @@ -52,6 +56,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 + - name: Install workflow contract parser + run: python3 -m pip install PyYAML==6.0.3 - name: Test release build orchestration run: python3 -m unittest discover -s dstack/build/shared/tests -v @@ -187,6 +193,32 @@ jobs: DSTACK_REV=${{ env.DSTACK_REV }} DSTACK_SRC_URL=${{ env.DSTACK_SRC_URL }} + - name: Verify offline one-shot rejection contract + run: | + python3 - <<'PYTEST' + import json + import pathlib + import subprocess + import tempfile + + with tempfile.TemporaryDirectory() as directory: + request = pathlib.Path(directory) / "invalid.json" + request.write_text(json.dumps({"attestation": "00"})) + result = subprocess.run( + ["docker", "run", "--rm", "--network=none", + "-v", f"{directory}:/evidence", "dstack-verifier-check:latest", + "--config", "/etc/dstack/dstack-verifier.toml", + "--verify", "/evidence/invalid.json"], + capture_output=True, text=True, timeout=30, + ) + assert result.returncode == 1, (result.returncode, result.stderr) + response = json.loads(result.stdout) + assert response["is_valid"] is False, response + assert isinstance(response["details"], dict), response + assert response["reason"], response + print("Real verifier CLI rejects malformed attestation with one JSON response") + PYTEST + - name: Verify pinned packages (runtime) run: | dstack/build/shared/verify-pinned-packages.sh dstack-verifier-check:latest \ diff --git a/.github/workflows/gateway-cluster-tests.yml b/.github/workflows/gateway-cluster-tests.yml index 74acca7f3..b238b8ad7 100644 --- a/.github/workflows/gateway-cluster-tests.yml +++ b/.github/workflows/gateway-cluster-tests.yml @@ -14,7 +14,7 @@ name: Gateway cluster tests # this replaces could not do. on: push: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] paths: - 'dstack/gateway/**' - 'dstack/cert-client/**' @@ -24,7 +24,7 @@ on: - 'dstack/crates/mock-attestation/**' - '.github/workflows/gateway-cluster-tests.yml' pull_request: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] paths: - 'dstack/gateway/**' - 'dstack/cert-client/**' diff --git a/.github/workflows/gateway-e2e-tests.yml b/.github/workflows/gateway-e2e-tests.yml index 38aa838fe..30aa77a0d 100644 --- a/.github/workflows/gateway-e2e-tests.yml +++ b/.github/workflows/gateway-e2e-tests.yml @@ -15,7 +15,7 @@ name: Gateway e2e tests # cluster mTLS path that a harness with the checks switched off cannot reach. on: push: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] paths: - 'dstack/gateway/**' - 'dstack/cert-client/**' @@ -27,7 +27,7 @@ on: - 'tools/mock-cf-dns/**' - '.github/workflows/gateway-e2e-tests.yml' pull_request: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] paths: - 'dstack/gateway/**' - 'dstack/cert-client/**' diff --git a/.github/workflows/gateway-proxy-tests.yml b/.github/workflows/gateway-proxy-tests.yml index 86413a4c1..eec0e950c 100644 --- a/.github/workflows/gateway-proxy-tests.yml +++ b/.github/workflows/gateway-proxy-tests.yml @@ -16,7 +16,7 @@ name: Gateway proxy tests # anything else was using TLS. on: push: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] paths: - 'dstack/gateway/**' - 'dstack/vendor/ktls/**' @@ -28,7 +28,7 @@ on: - 'sdk/simulator/**' - '.github/workflows/gateway-proxy-tests.yml' pull_request: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] paths: - 'dstack/gateway/**' - 'dstack/vendor/ktls/**' diff --git a/.github/workflows/gateway-release.yml b/.github/workflows/gateway-release.yml index 998209bc5..2f65daaf8 100644 --- a/.github/workflows/gateway-release.yml +++ b/.github/workflows/gateway-release.yml @@ -45,18 +45,22 @@ jobs: # Manual runs rebuild an existing release tag, not the selected branch. - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 with: ref: ${{ github.event_name == 'workflow_dispatch' && env.RELEASE_TAG || github.sha }} - - name: Log in to Docker Hub - uses: docker/login-action@v3 + - name: Normalise registry owner + run: echo "OWNER_LC=${GITHUB_REPOSITORY_OWNER,,}" >> "$GITHUB_ENV" + + - name: Log in to GitHub Container Registry + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: - username: ${{ vars.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Resolve the revision to build run: echo "GIT_REV=$(git rev-parse HEAD)" >> "$GITHUB_ENV" @@ -71,7 +75,7 @@ jobs: METADATA_FILE: ${{ runner.temp }}/build-metadata.json GIT_REV: ${{ env.GIT_REV }} IMAGE_VERSION: ${{ env.VERSION }} - IMAGE_NAME: ${{ vars.DOCKERHUB_ORG }}/dstack-gateway:${{ env.VERSION }} + IMAGE_NAME: ghcr.io/${{ env.OWNER_LC }}/dstack-gateway:${{ env.VERSION }} IMAGE_SOURCE_URL: ${{ github.server_url }}/${{ github.repository }} DSTACK_SRC_URL: ${{ github.server_url }}/${{ github.repository }}.git run: | @@ -84,15 +88,27 @@ jobs: fi echo "digest=$digest" >> "$GITHUB_OUTPUT" + - name: Verify anonymous pull access + env: + IMAGE_DIGEST: ${{ steps.build-and-push.outputs.digest }} + run: | + token=$(curl -fsSL \ + "https://ghcr.io/token?scope=repository:${OWNER_LC}/dstack-gateway:pull" \ + | jq -er .token) + curl -fsSL -o /dev/null \ + -H "Authorization: Bearer $token" \ + -H "Accept: application/vnd.oci.image.manifest.v1+json" \ + "https://ghcr.io/v2/${OWNER_LC}/dstack-gateway/manifests/$IMAGE_DIGEST" + - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 + uses: actions/attest-build-provenance@92c65d2898f1f53cfdc910b962cecff86e7f8fcc # v1 with: - subject-name: "docker.io/${{ vars.DOCKERHUB_ORG }}/dstack-gateway" + subject-name: "ghcr.io/${{ env.OWNER_LC }}/dstack-gateway" subject-digest: ${{ steps.build-and-push.outputs.digest }} push-to-registry: true - name: GitHub Release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 with: tag_name: ${{ env.RELEASE_TAG }} target_commitish: ${{ env.GIT_REV }} @@ -100,7 +116,7 @@ jobs: body: | ## Docker Image Information - **Image**: `docker.io/${{ vars.DOCKERHUB_ORG }}/dstack-gateway:${{ env.VERSION }}` + **Image**: `ghcr.io/${{ env.OWNER_LC }}/dstack-gateway:${{ env.VERSION }}` **Digest (SHA256)**: `${{ steps.build-and-push.outputs.digest }}` diff --git a/.github/workflows/kms-release.yml b/.github/workflows/kms-release.yml index 122d886b6..65756f6e9 100644 --- a/.github/workflows/kms-release.yml +++ b/.github/workflows/kms-release.yml @@ -45,7 +45,7 @@ jobs: # Manual runs rebuild an existing release tag, not the selected branch. - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 with: ref: ${{ github.event_name == 'workflow_dispatch' && env.RELEASE_TAG || github.sha }} @@ -56,14 +56,18 @@ jobs: dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable \ dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades - - name: Log in to Docker Hub - uses: docker/login-action@v3 + - name: Normalise registry owner + run: echo "OWNER_LC=${GITHUB_REPOSITORY_OWNER,,}" >> "$GITHUB_ENV" + + - name: Log in to GitHub Container Registry + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: - username: ${{ vars.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Resolve the revision to build run: echo "GIT_REV=$(git rev-parse HEAD)" >> "$GITHUB_ENV" @@ -78,7 +82,7 @@ jobs: METADATA_FILE: ${{ runner.temp }}/build-metadata.json GIT_REV: ${{ env.GIT_REV }} IMAGE_VERSION: ${{ env.VERSION }} - IMAGE_NAME: ${{ vars.DOCKERHUB_ORG }}/dstack-kms:${{ env.VERSION }} + IMAGE_NAME: ghcr.io/${{ env.OWNER_LC }}/dstack-kms:${{ env.VERSION }} IMAGE_SOURCE_URL: ${{ github.server_url }}/${{ github.repository }} DSTACK_SRC_URL: ${{ github.server_url }}/${{ github.repository }}.git run: | @@ -91,24 +95,35 @@ jobs: fi echo "digest=$digest" >> "$GITHUB_OUTPUT" + - name: Verify anonymous pull access + env: + IMAGE_DIGEST: ${{ steps.build-and-push.outputs.digest }} + run: | + token=$(curl -fsSL \ + "https://ghcr.io/token?scope=repository:${OWNER_LC}/dstack-kms:pull" \ + | jq -er .token) + curl -fsSL -o /dev/null \ + -H "Authorization: Bearer $token" \ + -H "Accept: application/vnd.oci.image.manifest.v1+json" \ + "https://ghcr.io/v2/${OWNER_LC}/dstack-kms/manifests/$IMAGE_DIGEST" + - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 + uses: actions/attest-build-provenance@92c65d2898f1f53cfdc910b962cecff86e7f8fcc # v1 with: - subject-name: "docker.io/${{ vars.DOCKERHUB_ORG }}/dstack-kms" + subject-name: "ghcr.io/${{ env.OWNER_LC }}/dstack-kms" subject-digest: ${{ steps.build-and-push.outputs.digest }} push-to-registry: true - name: Install Foundry - uses: foundry-rs/foundry-toolchain@v1 + uses: foundry-rs/foundry-toolchain@908c540300062bd5a7e473851cdb4282204cee09 # v1 - name: Compile contracts with Foundry run: | cd dstack/kms/auth-eth - forge install forge build - name: GitHub Release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 with: tag_name: ${{ env.RELEASE_TAG }} target_commitish: ${{ env.GIT_REV }} @@ -119,7 +134,7 @@ jobs: body: | ## Docker Image Information - **Image**: `docker.io/${{ vars.DOCKERHUB_ORG }}/dstack-kms:${{ env.VERSION }}` + **Image**: `ghcr.io/${{ env.OWNER_LC }}/dstack-kms:${{ env.VERSION }}` **Digest (SHA256)**: `${{ steps.build-and-push.outputs.digest }}` diff --git a/.github/workflows/local-key-provider-release.yml b/.github/workflows/local-key-provider-release.yml index 3782bb4b5..d4cd4d118 100644 --- a/.github/workflows/local-key-provider-release.yml +++ b/.github/workflows/local-key-provider-release.yml @@ -25,48 +25,76 @@ jobs: build-and-release: runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Parse version from tag + - name: Parse and check version env: DISPATCH_VERSION: ${{ inputs.version }} run: | - if [[ "$GITHUB_REF" == refs/tags/local-key-provider-v* ]]; then - VERSION=${GITHUB_REF#refs/tags/local-key-provider-v} - else + if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then VERSION=$DISPATCH_VERSION + else + case "$GITHUB_REF" in + refs/tags/local-key-provider-v*) VERSION=${GITHUB_REF#refs/tags/local-key-provider-v} ;; + *) echo "Expected a local-key-provider-v* tag" >&2; exit 1 ;; + esac + fi + if [[ ! "$VERSION" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "Version must be a valid Docker tag" >&2 + exit 1 fi echo "VERSION=$VERSION" >> "$GITHUB_ENV" - echo "Parsed version: $VERSION" + echo "RELEASE_TAG=local-key-provider-v$VERSION" >> "$GITHUB_ENV" + + # Manual runs rebuild an existing release tag, not the selected branch. + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && env.RELEASE_TAG || github.sha }} - # ghcr.io requires a lowercase owner; the org is mixed-case. - name: Normalise registry owner run: echo "OWNER_LC=${GITHUB_REPOSITORY_OWNER,,}" >> "$GITHUB_ENV" - name: Log in to GitHub Container Registry - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + + - name: Resolve the revision to build + run: echo "GIT_REV=$(git rev-parse HEAD)" >> "$GITHUB_ENV" - name: Build and push Docker image id: build-and-push - uses: docker/build-push-action@v7 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 with: context: dstack file: dstack/local-key-provider/build/Dockerfile.key-provider push: true tags: ghcr.io/${{ env.OWNER_LC }}/local-key-provider:${{ env.VERSION }} + labels: | + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ env.GIT_REV }} + org.opencontainers.image.version=${{ env.VERSION }} platforms: linux/amd64 provenance: false + - name: Verify anonymous pull access + env: + IMAGE_DIGEST: ${{ steps.build-and-push.outputs.digest }} + run: | + token=$(curl -fsSL \ + "https://ghcr.io/token?scope=repository:${OWNER_LC}/local-key-provider:pull" \ + | jq -er .token) + curl -fsSL -o /dev/null \ + -H "Authorization: Bearer $token" \ + -H "Accept: application/vnd.oci.image.manifest.v1+json" \ + "https://ghcr.io/v2/${OWNER_LC}/local-key-provider/manifests/$IMAGE_DIGEST" + - name: Generate artifact attestation - uses: actions/attest-build-provenance@v4 + uses: actions/attest-build-provenance@8beda2b7ed98355c0e97c0a63bec38ae472e66c4 # v4 with: subject-name: "ghcr.io/${{ env.OWNER_LC }}/local-key-provider" subject-digest: ${{ steps.build-and-push.outputs.digest }} @@ -91,10 +119,11 @@ jobs: echo "MRSIGNER=$MRSIGNER" >> "$GITHUB_ENV" - name: GitHub Release - uses: softprops/action-gh-release@v3 + uses: softprops/action-gh-release@5113cdc90fd4d541c801c55356214017bf5ae34b # v3 with: name: "Local Key Provider Release v${{ env.VERSION }}" - tag_name: local-key-provider-v${{ env.VERSION }} + tag_name: ${{ env.RELEASE_TAG }} + target_commitish: ${{ env.GIT_REV }} files: local-key-provider.sig.json body: | ## Docker Image Information diff --git a/.github/workflows/mkosi-build.yml b/.github/workflows/mkosi-build.yml index 258cee143..f14b0f263 100644 --- a/.github/workflows/mkosi-build.yml +++ b/.github/workflows/mkosi-build.yml @@ -25,7 +25,7 @@ on: # would be silently dropped. The static job costs seconds and the image build # is gated by its own `if`, so an unfiltered push trigger is cheap. push: - branches: [next, 'release/**'] + branches: [alpha, next, 'release/**'] tags: ['mkosi-os-v*'] concurrency: @@ -63,6 +63,7 @@ jobs: os_image_hash: ${{ steps.hashes.outputs.os_image_hash }} bare_sha256: ${{ steps.hashes.outputs.bare_sha256 }} uki_sha256: ${{ steps.hashes.outputs.uki_sha256 }} + kernel_devel_sha256: ${{ steps.hashes.outputs.kernel_devel_sha256 }} steps: # GitHub-hosted runners have enough CPU and memory for this build, but @@ -206,6 +207,7 @@ jobs: bare_sha256=$(sha256sum "$bare" | awk '{print $1}') uki_sha256=$(sha256sum "$uki" | awk '{print $1}') + kernel_devel_sha256=$(sha256sum "$DIST_DIR/dstack-$version-kernel-devel.tar.gz" | awk '{print $1}') git_revision=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["git_revision"])' "$img/metadata.json") mrtd=$("$DSTACK_MR" inspect-measurement "$img/measurement.tdx.cbor" \ | python3 -c 'import json,sys; print(json.load(sys.stdin)["tdvf"]["mrtd"]["single_pass"])') @@ -219,6 +221,7 @@ jobs: echo "os_image_hash=$os_image_hash" echo "bare_sha256=$bare_sha256" echo "uki_sha256=$uki_sha256" + echo "kernel_devel_sha256=$kernel_devel_sha256" } >> "$GITHUB_OUTPUT" # os_image_hash is the value operators need most, so surface it in the @@ -236,6 +239,7 @@ jobs: echo "| AWS \`boot_pcr_digest\` | \`$aws_pcr\` |" echo "| \`dstack-$version.tar.gz\` | \`$bare_sha256\` |" echo "| \`dstack-$version-uki.tar.gz\` | \`$uki_sha256\` |" + echo "| \`dstack-$version-kernel-devel.tar.gz\` | \`$kernel_devel_sha256\` |" echo echo '
sha256sum.txt (os_image_hash preimage)' echo @@ -262,6 +266,7 @@ jobs: echo "aws_boot_pcr $aws_pcr" echo "$bare_sha256 dstack-$version.tar.gz" echo "$uki_sha256 dstack-$version-uki.tar.gz" + echo "$kernel_devel_sha256 dstack-$version-kernel-devel.tar.gz" } | tee "$DIST_DIR/image-hashes.txt" - name: Upload guest images @@ -274,11 +279,54 @@ jobs: retention-days: 30 if-no-files-found: error + kernel-builder: + name: Kernel module builder image + if: >- + github.event_name == 'workflow_dispatch' || + startsWith(github.ref, 'refs/tags/mkosi-os-v') + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Download guest images + uses: actions/download-artifact@v4 + with: + name: mkosi-guest-images + path: dist + + - name: Build the builder image + env: + VERSION: ${{ needs.build.outputs.version }} + run: | + owner=$(echo "$GITHUB_REPOSITORY_OWNER" | tr '[:upper:]' '[:lower:]') + echo "IMAGE=ghcr.io/$owner/dstack-kernel-builder:$VERSION" >> "$GITHUB_ENV" + GIT_REVISION="$GITHUB_SHA" ./os/image/kernel-builder/build.sh \ + "dist/dstack-$VERSION-kernel-devel.tar.gz" "ghcr.io/$owner/dstack-kernel-builder:$VERSION" + + - name: Log in to GitHub Container Registry + if: startsWith(github.ref, 'refs/tags/mkosi-os-v') + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push the builder image + if: startsWith(github.ref, 'refs/tags/mkosi-os-v') + run: docker push "$IMAGE" + release: name: Publish release # The tag is the publish decision. Nothing else reaches this job. if: startsWith(github.ref, 'refs/tags/mkosi-os-v') - needs: build + needs: [build, kernel-builder] runs-on: ubuntu-latest timeout-minutes: 30 environment: release @@ -339,6 +387,7 @@ jobs: OS_IMAGE_HASH: ${{ needs.build.outputs.os_image_hash }} BARE_SHA256: ${{ needs.build.outputs.bare_sha256 }} UKI_SHA256: ${{ needs.build.outputs.uki_sha256 }} + KERNEL_DEVEL_SHA256: ${{ needs.build.outputs.kernel_devel_sha256 }} run: | set -euo pipefail { @@ -353,9 +402,14 @@ jobs: echo "| Git revision | \`$GITHUB_SHA\` |" echo "| \`dstack-$VERSION.tar.gz\` | \`$BARE_SHA256\` |" echo "| \`dstack-$VERSION-uki.tar.gz\` | \`$UKI_SHA256\` |" + echo "| \`dstack-$VERSION-kernel-devel.tar.gz\` | \`$KERNEL_DEVEL_SHA256\` |" echo echo "\`image-hashes.txt\` additionally records the TDX \`MRTD\`, the GCP" echo "\`uki_auth\` hash and the AWS \`boot_pcr_digest\` for this image." + echo + echo "To build out-of-tree kernel modules, use" + echo "\`ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/dstack-kernel-builder:$VERSION\`; see" + echo "[docs/building-guest-os.md](https://github.com/${GITHUB_REPOSITORY}/blob/${TAG}/docs/building-guest-os.md#kernel-headers-for-out-of-tree-modules)." } > release-notes.md # --verify-tag: the tag is the publish trigger, so it must already # exist. This job never creates one. @@ -371,6 +425,7 @@ jobs: gh release create "$TAG" \ "dist/dstack-$VERSION.tar.gz" \ "dist/dstack-$VERSION-uki.tar.gz" \ + "dist/dstack-$VERSION-kernel-devel.tar.gz" \ dist/image-hashes.txt \ --title "dstack OS v$VERSION" \ --notes-file release-notes.md \ diff --git a/.github/workflows/prek-check.yml b/.github/workflows/prek-check.yml index 267f21c55..78cb11603 100644 --- a/.github/workflows/prek-check.yml +++ b/.github/workflows/prek-check.yml @@ -6,9 +6,9 @@ name: Prek checks on: push: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] pull_request: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] permissions: contents: read diff --git a/.github/workflows/qemu-acpi-differential.yml b/.github/workflows/qemu-acpi-differential.yml index de8dea222..d658fc013 100644 --- a/.github/workflows/qemu-acpi-differential.yml +++ b/.github/workflows/qemu-acpi-differential.yml @@ -5,7 +5,7 @@ name: QEMU ACPI differential tests on: pull_request: - branches: [next] + branches: [alpha, next] paths: - .github/workflows/qemu-acpi-differential.yml - dstack/crates/qemu-acpi/** diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 0cb18deec..0542c9e58 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -6,9 +6,9 @@ name: Rust checks on: push: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] pull_request: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/sdk-compat.yaml b/.github/workflows/sdk-compat.yaml index 2fdc127a6..096f59965 100644 --- a/.github/workflows/sdk-compat.yaml +++ b/.github/workflows/sdk-compat.yaml @@ -17,14 +17,14 @@ permissions: # rest silently uncovered the moment one of them changed behaviour. on: push: - branches: [next, 'release/**'] + branches: [alpha, next, 'release/**'] paths: - 'sdk/**' - 'dstack/**' - 'rust-toolchain.toml' - '.github/workflows/sdk-compat.yaml' pull_request: - branches: [next, 'release/**'] + branches: [alpha, next, 'release/**'] paths: - 'sdk/**' - 'dstack/**' diff --git a/.github/workflows/sdk.yaml b/.github/workflows/sdk.yaml index 550806ff9..2de0e3b00 100644 --- a/.github/workflows/sdk.yaml +++ b/.github/workflows/sdk.yaml @@ -9,9 +9,9 @@ permissions: on: push: - branches: [next, 'release/**'] + branches: [alpha, next, 'release/**'] pull_request: - branches: [next, 'release/**'] + branches: [alpha, next, 'release/**'] env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/spdx-check.yml b/.github/workflows/spdx-check.yml index dc24dfd08..e896f0a1e 100644 --- a/.github/workflows/spdx-check.yml +++ b/.github/workflows/spdx-check.yml @@ -6,9 +6,9 @@ name: SPDX License Check on: push: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] pull_request: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] jobs: reuse-lint: diff --git a/.github/workflows/verifier-release.yml b/.github/workflows/verifier-release.yml index 857a6382c..c41ed1198 100644 --- a/.github/workflows/verifier-release.yml +++ b/.github/workflows/verifier-release.yml @@ -45,18 +45,22 @@ jobs: # Manual runs rebuild an existing release tag, not the selected branch. - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 with: ref: ${{ github.event_name == 'workflow_dispatch' && env.RELEASE_TAG || github.sha }} - - name: Log in to Docker Hub - uses: docker/login-action@v3 + - name: Normalise registry owner + run: echo "OWNER_LC=${GITHUB_REPOSITORY_OWNER,,}" >> "$GITHUB_ENV" + + - name: Log in to GitHub Container Registry + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: - username: ${{ vars.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Resolve the revision to build run: echo "GIT_REV=$(git rev-parse HEAD)" >> "$GITHUB_ENV" @@ -71,8 +75,8 @@ jobs: METADATA_FILE: ${{ runner.temp }}/build-metadata.json GIT_REV: ${{ env.GIT_REV }} IMAGE_VERSION: ${{ env.VERSION }} - IMAGE_NAME: ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:${{ env.VERSION }} - LATEST_IMAGE_NAME: ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:latest + IMAGE_NAME: ghcr.io/${{ env.OWNER_LC }}/dstack-verifier:${{ env.VERSION }} + LATEST_IMAGE_NAME: ghcr.io/${{ env.OWNER_LC }}/dstack-verifier:latest IMAGE_SOURCE_URL: ${{ github.server_url }}/${{ github.repository }} DSTACK_SRC_URL: ${{ github.server_url }}/${{ github.repository }}.git run: | @@ -86,15 +90,27 @@ jobs: fi echo "digest=$digest" >> "$GITHUB_OUTPUT" + - name: Verify anonymous pull access + env: + IMAGE_DIGEST: ${{ steps.build-and-push.outputs.digest }} + run: | + token=$(curl -fsSL \ + "https://ghcr.io/token?scope=repository:${OWNER_LC}/dstack-verifier:pull" \ + | jq -er .token) + curl -fsSL -o /dev/null \ + -H "Authorization: Bearer $token" \ + -H "Accept: application/vnd.oci.image.manifest.v1+json" \ + "https://ghcr.io/v2/${OWNER_LC}/dstack-verifier/manifests/$IMAGE_DIGEST" + - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 + uses: actions/attest-build-provenance@92c65d2898f1f53cfdc910b962cecff86e7f8fcc # v1 with: - subject-name: "docker.io/${{ vars.DOCKERHUB_ORG }}/dstack-verifier" + subject-name: "ghcr.io/${{ env.OWNER_LC }}/dstack-verifier" subject-digest: ${{ steps.build-and-push.outputs.digest }} push-to-registry: true - name: GitHub Release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 with: tag_name: ${{ env.RELEASE_TAG }} target_commitish: ${{ env.GIT_REV }} @@ -102,7 +118,7 @@ jobs: body: | ## Docker Image Information - **Image**: `docker.io/${{ vars.DOCKERHUB_ORG }}/dstack-verifier:${{ env.VERSION }}` + **Image**: `ghcr.io/${{ env.OWNER_LC }}/dstack-verifier:${{ env.VERSION }}` **Digest (SHA256)**: `${{ steps.build-and-push.outputs.digest }}` diff --git a/.github/workflows/vmm-ui.yml b/.github/workflows/vmm-ui.yml index 259865541..3d30d7f24 100644 --- a/.github/workflows/vmm-ui.yml +++ b/.github/workflows/vmm-ui.yml @@ -11,12 +11,12 @@ permissions: # workflow-level filter is safe. on: push: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] paths: - 'dstack/vmm/ui/**' - '.github/workflows/vmm-ui.yml' pull_request: - branches: [ next, 'release/**' ] + branches: [ alpha, next, 'release/**' ] paths: - 'dstack/vmm/ui/**' - '.github/workflows/vmm-ui.yml' diff --git a/CHANGELOG.md b/CHANGELOG.md index f94c67e96..a08edfc36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - http-client: a caller can bound the response body (`http_request_bounded`, `PrpcClient::with_max_response_bytes`). Nothing is bounded by default — `dstack vmm logs --lines 100000` is a legitimate multi-megabyte fetch — but every client that talks to a guest agent opts in, in the gateway and in the VMM, because a CVM is untrusted and one of them polls on a timer against the whole fleet ### Fixed +- sdk: Python and JavaScript blockchain adapters now reject TLS-key responses. The deprecated conversion path read fixed PKCS#8 framing as private-key bytes, causing different TLS keys to derive the same Ethereum and Solana wallets. Use `get_key()` / `getKey()` for wallet keys. - data disks: discard now propagates through ZFS or ext4, dm-crypt, virtio-blk, and QEMU so encrypted qcow2 images release deleted blocks instead of growing with lifetime writes. Discard defaults on and can be disabled with `storage_discard: false` when allocation-pattern leakage is unacceptable; upgrading an existing ZFS pool also starts a one-time trim for historical free space - certbot: a certificate covering both a name and its wildcard (`example.com` and `*.example.com`) could never be issued over dns-01. The two authorizations are answered under one `_acme-challenge.example.com`, each with its own TXT value, and the publish step cleared every TXT record at that name before writing its own -- so the second authorization deleted the record answering the first, and the order failed with `Correct value not found for DNS challenge`. Clearing leftovers from an aborted run is now done once per challenge name per issuance, and the records for one name accumulate instead of replacing each other; cleanup afterwards is unchanged, deleting each record this run created by id - certbot: editing `domains` in `certbot.toml` had no effect once a certificate existed. Issuance was skipped whenever `live/cert.pem` was present, whatever names it carried, and renewal read its name list back off that certificate rather than the configuration -- so an added or removed name never reached the CA, and the mismatch survived every renewal. The live certificate's DNS names are now compared against the configured list (as sets, case- and trailing-dot-insensitive) and a mismatch reissues, logging both lists. A reissue that fails does not take the renewal check down with it: a name the CA will not validate is reported on every cycle, while the certificate actually being served keeps renewing, and the failure is still what the run returns unless the renewal committed something of its own diff --git a/docs/attestation-nitro-enclave.md b/docs/attestation-nitro-enclave.md index 1173cb758..7e0df262b 100644 --- a/docs/attestation-nitro-enclave.md +++ b/docs/attestation-nitro-enclave.md @@ -59,6 +59,33 @@ The verifier returns `DstackVerifiedReport::DstackNitroEnclave` containing: - `user_data` (report_data) - `timestamp` +## KMS key release + +Key release to a Nitro Enclave is gated locally by `nitro_enclave_key_release` +in `kms.toml`, which defaults to `false`: + +```toml +[core] +nitro_enclave_key_release = true +``` + +This gate sits *after* full attestation verification and after the external auth +policy decision, so both must also pass. It mirrors `sev_snp_key_release` and +`aws_nitro_tpm_key_release`: platforms whose confidentiality guarantee does not +come from a CPU the way Intel TDX's does are opt-in, so that enabling one never +implies another. + +Two properties of this platform are worth knowing before enabling it: + +- The AWS hypervisor is in the TCB. The NSM document is signed by an AWS root, + and PCR0/1/2 measure the enclave image, but there is no hardware memory + encryption isolating the enclave from the platform operator. +- dstack extends no runtime measurement register on this variant, so `app_id` + and `instance_id` in the attestation come from an event list that nothing + replays against a quoted register. `os_image_hash` and `compose_hash`, which + are derived from the signed PCRs, are unaffected. + ## Relevant Code - `dstack-attest/src/attestation.rs` - `nsm-qvl/src/verify.rs` +- `kms/src/main_service.rs` (`ensure_key_release_allowed`) diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index d319a7827..7007d4c86 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -171,7 +171,8 @@ needs an `allow` line for the bridge. - `netd` creates a persistent TAP, attaches it to the bridge, binds the nwfilter if the node filters, and the VMM passes `-netdev tap,id=net0,ifname=,...` - Guest MAC address is derived from SHA256 of the VM ID, with an optional configurable prefix (stable across restarts for DHCP IP consistency) - The host DHCP server (dnsmasq) assigns an IP to the VM -- The TAP outlives QEMU and is deleted when the VMM tears the VM's networking down +- The TAP outlives QEMU and is deleted when the VMM tears the VM's networking down, so a VM that crashes does not leave its filter rules attached to a name the next VM could take +- Every interface `netd` creates records which VM of which VMM instance it belongs to, in the kernel's interface alias — see [Who owns an interface](#who-owns-an-interface) - The VMM process needs neither root nor `CAP_NET_ADMIN`; `netd` holds that privilege in a separate service ### MAC address prefix @@ -237,6 +238,76 @@ told. An unpinned mapping goes to the first user-mode NIC; a VM that has none is not refused — it may have been deployed before this — but every mapping it strands is named in the launch log. +## Who owns an interface + +`netd` names an interface `dt<12 hex>`, a digest of (VMM instance, VM, NIC +index). That answers "where is this VM's interface" but not "whose is this +interface" — and the second question is the one a leaked interface poses. So +`netd` also records the identity on the interface itself: + +```console +$ ip -d link show dtc41d9e0b7a52 | grep alias + alias dstack1:0:path-3f9a1c8e7d2b4a60:0a1b2c3d4e5f6071 +``` + +The kernel holds that for exactly the interface's lifetime, so unlike a file on +disk it cannot be written late, lost, or left behind. It is a hint, never an +authority: a record is believed only when re-deriving the interface name from +it reproduces the name it is written on, so a forged, truncated or ambiguous +record reads the same as no record at all. + +Teardown does not need it — a sweep derives the names it deletes. What needs it +is an operator, and a host running several VMM instances, where it is the only +thing that tells one instance's interfaces from another's. + +```bash +# What netd holds on this host +sudo dstack-vmm netd list + +# Everything one VM holds, for a VM whose VMM will never ask again +sudo dstack-vmm netd remove-vm --instance path-3f9a1c8e7d2b4a60 --vm 0a1b2c3d4e5f6071 +``` + +### When a release does not land + +The VMM writes `.netd-pending` into the VM's directory before it asks `netd` +to build anything, and deletes it only after `netd` has swept every interface +that VM could hold. The marker is the only record the VMM keeps of what `netd` +may hold, so it survives failed launches and changes to a topology that no +longer uses `netd`. A VM without it has nothing to release, and its stop, +removal and launch never contact `netd` for one. + +A sweep derives each of the 256 names the VM's identity could produce, so it +needs no record of which NICs existed: it also reaches an interface a crash left +behind, or one whose NIC the manifest has since dropped. It is idempotent, and +every path that releases a VM retries it until it lands: + +- a launch sweeps before it prepares; +- a stop that could not reach `netd` leaves the marker, and the VMM checks + stopped VMs with a marker every minute and sweeps them; +- a removal keeps the VM's directory, and the VM marked as being removed, and + retries the sweep with backoff until `netd` answers. The directory is deleted + after that, with `.removing` last, so a removal interrupted by a crash is + resumed by the next VMM start. + +On an unfiltered node, an unavailable `libvirtd` does not make an otherwise +successful TAP sweep fail. Filtered nodes still require confirmation that their +nwfilter bindings have been released; deleting the TAP alone is not sufficient. + +What no VMM will retry is an interface whose VM directory an operator deleted +by hand, or one recorded under an instance ID no VMM uses any more. `netd list` +shows both, with the instance and VM they are recorded under: + +```bash +sudo dstack-vmm netd list +sudo dstack-vmm netd remove-vm --instance --vm +sudo dstack-vmm netd remove-interface dtc41d9e0b7a52 +``` + +Changing `cvm.instance_id` — or `run_path`, which it is derived from — strands +interfaces the same way. Running VMs keep working until they stop, and +`netd list` still shows the old instance ID, which is what `remove-vm` needs. + ### Mixing networking modes Bridge and user-mode VMs can coexist. Set the global default in `vmm.toml` and override per-VM as needed: diff --git a/docs/building-guest-os.md b/docs/building-guest-os.md index 2480b7cc1..4521380eb 100644 --- a/docs/building-guest-os.md +++ b/docs/building-guest-os.md @@ -93,7 +93,8 @@ Release artifacts are written under: os/mkosi/repro-build/build/out/prod/ ├── dstack-/ ├── dstack-.tar.gz -└── dstack--uki.tar.gz +├── dstack--uki.tar.gz +└── dstack--kernel-devel.tar.gz ``` To use a different build directory, run @@ -103,7 +104,9 @@ prints the `os_image_hash` and the SHA-256 of both archives. The bare-metal archive includes the kernel, initramfs, OVMF firmware, partitioned dm-verity rootfs, platform measurement CBOR files, `sha256sum.txt`, `digest.txt`, and `metadata.json`. The UKI archive -includes the bootable `disk.raw` plus its identity and measurement files. +includes the bootable `disk.raw` plus its identity and measurement files. The +kernel-devel archive is the build tree for out-of-tree modules; it is not part +of the image identity (see below). Inspect and verify an archive with: @@ -127,6 +130,34 @@ FLAVORS="prod dev" ./os/mkosi/repro-build/repro-build.sh The development archive is written to `out/dev/dstack-dev-.tar.gz` and records `"is_dev": true` in `metadata.json`. +## Kernel headers for out-of-tree modules + +Each release publishes the guest kernel's build tree for building out-of-tree +modules: + +- `dstack--kernel-devel.tar.gz`, the exported build tree, and +- `ghcr.io/dstack-tee/dstack-kernel-builder:`, the same tree plus a + compiler from the pinned Debian snapshot the kernel was built with, with + `KDIR` set. + +```dockerfile +FROM ghcr.io/dstack-tee/dstack-kernel-builder: AS kmod +COPY mymod/ /src +RUN make -C $KDIR M=/src modules + +FROM alpine +COPY --from=kmod /src/mymod.ko /opt/ +``` + +To build the image locally from a release archive, run +`os/image/kernel-builder/build.sh `. + +Neither artifact is part of `sha256sum.txt`, so they do not affect +`os_image_hash`. A module loaded by an application (which needs +`CAP_SYS_MODULE`) is covered by the container image digest and therefore the +compose hash, but it is not measured separately and dstack does not enforce +module signatures. + ## Check reproducibility For a release candidate: diff --git a/docs/libvirt-network-filter.md b/docs/libvirt-network-filter.md index 1e161c183..97e659c0c 100644 --- a/docs/libvirt-network-filter.md +++ b/docs/libvirt-network-filter.md @@ -20,6 +20,11 @@ The measurable acceptance criteria are: that TAP carries a binding. - `network_filter = "libvirt"` creates the TAP and filter binding before QEMU is submitted to Supervisor, and uses QEMU `-netdev tap`. +- An nwfilter binding outlives the TAP it was bound to, so a teardown clears + the binding at every name that VM could have used, whether or not the + interface is still there. `dstack-vmm netd list` shows a binding whose + interface is already gone as a `binding` row; remove one with + `dstack-vmm netd remove-interface `. - A failed TAP or filter setup prevents QEMU from starting and rolls back all interfaces prepared for that VM. - Normal stop and removal delete the filter binding and TAP. @@ -105,9 +110,26 @@ Deploy VMM instances that do not share this trust boundary with dedicated netd sockets and distinct filesystem permissions. `netd` invokes fixed absolute `ip` and `virsh` executables with separate -arguments. It never accepts a command, executable path, TAP name, or raw XML -from a client. Filter XML is generated internally with XML escaping and is -validated by libvirt. +arguments. It never accepts a command, executable path, or raw XML from a +client. The only TAP name it accepts is the one `RemoveInterfaceByName` takes, +and only a name of the form netd itself creates (`dt` and 12 lowercase hex +digits), on a tun/tap or macvtap device. Filter XML is generated internally with +XML escaping and is validated by libvirt. + +Teardown by identity only reaches the NIC indices its caller still has a record +of, and that record is written *after* the interface exists — a VMM killed in +between leaves a TAP nothing on disk points at, and a manifest that lost a NIC +leaves the same thing behind. `RemoveVm` names a VM instead of an interface +and derives every name that VM could occupy, so neither has to be recorded for +teardown to work. The VMM keeps a single `.netd-pending` marker per VM from +before its first prepare until a sweep succeeds, and sweeps whenever that marker +is present: before a launch, on stop, on removal, and periodically for stopped +VMs. See [bridge-networking.md](bridge-networking.md#when-a-release-does-not-land). + +A prepare also carries `workdir`, which `netd` does not need to build the TAP. +It names the VM's directory on the host: untrusted, never read for a decision, +and present only so an operator reading `netd`'s log can get from an opaque TAP +name back to the VM. ## Deployment modes @@ -199,8 +221,10 @@ sudo dstack-vmm --config ./vmm.toml \ ``` User networking and a caller-supplied netdev never ask `netd` to build an -interface. Bridge and macvtap do ask, and fail closed if `netd` is -unavailable. +interface, and a VM that has never used `netd` never contacts it. A VM that +moved from bridge or macvtap to user networking still releases what it held, +but nothing about its launch depends on the answer. Bridge and macvtap do ask, +and fail closed if `netd` is unavailable. Filtered TAP netdevs follow the node's `vhost` and `queues` settings like any other TAP-backed NIC (see [network-data-plane.md](network-data-plane.md)). The diff --git a/docs/rc-testing-runbook.md b/docs/rc-testing-runbook.md index 1e4478679..971a451ed 100644 --- a/docs/rc-testing-runbook.md +++ b/docs/rc-testing-runbook.md @@ -99,8 +99,8 @@ the CLI's default `--url`. On both machines used last time it pointed at a live deployment with credentials, so any `deploy` that omitted `--url` would have landed there. **Pass `--url http://127.0.0.1:` on every single invocation.** -**Component tags publish to Docker Hub.** `kms-v*`, `gateway-v*`, `verifier-v*` -and friends trigger release workflows that push to `${DOCKERHUB_ORG}` and cut +**Component tags publish to GHCR.** `kms-v*`, `gateway-v*`, `verifier-v*` +and friends trigger release workflows that push to `ghcr.io/dstack-tee` and cut GitHub releases. If you only want images in a private registry, build locally and push by hand — do not push those tags. @@ -362,7 +362,8 @@ start changing things. |---|---|---|---| | `[cvm] platform` | `vmm.toml` (host) | `"auto"` | `"auto"` picks SEV-SNP when the host CPU flags contain `sev_snp`; set `"amd-sev-snp"` to be explicit and to fail loudly on the wrong host | | `sev_snp_key_release` | `kms.toml`, `[core]` | `false` | **`true`**, or no AMD guest ever gets a key | -| `aws_nitro_tpm_key_release` | `kms.toml`, `[core]` | `false` | leave off; listed because it is the only other gate of this shape | +| `aws_nitro_tpm_key_release` | `kms.toml`, `[core]` | `false` | leave off; listed because it is another gate of this shape | +| `nitro_enclave_key_release` | `kms.toml`, `[core]` | `false` | leave off; listed because it is another gate of this shape | | `[core.attestation.urls] amd_kds` | `kms.toml`, `dstack-verifier.toml` | `https://kdsintf.amd.com/vcek/v1` | override only to point at a cache or mirror | | `[core.attestation.root_ca] sev_snp_milan` / `_genoa` / `_turin` | same | unset — vendor ARK compiled in | leave unset unless you are testing against a non-production root | | `insecure_allow_external_trust_anchors` | same, `[core.attestation]` | `false` | must be `true` if *any* `root_ca` path above is set | diff --git a/docs/security/README.md b/docs/security/README.md index 4086f2f79..d260325d5 100644 --- a/docs/security/README.md +++ b/docs/security/README.md @@ -6,6 +6,7 @@ Use these resources to understand dstack's trust model, production requirements, - [Security Model](./security-model.md) - threat model, trust boundaries, and verifier checklist - [Security Best Practices](./security-best-practices.md) - production hardening for KMS, gateway, and VMM deployments +- [App Key Compromise Recovery](./app-key-compromise-recovery.md) - containment and identity replacement when app keys may be disclosed - [Security Audit](./dstack-audit.pdf) - zkSecurity audit report - [Public Security Reports](./public-security-reports.md) - status of already-public reports and findings - [CVM Boundaries](./cvm-boundaries.md) - data exchanged across the CVM, host, KMS, and gateway diff --git a/docs/security/app-key-compromise-recovery.md b/docs/security/app-key-compromise-recovery.md new file mode 100644 index 000000000..14052f9e2 --- /dev/null +++ b/docs/security/app-key-compromise-recovery.md @@ -0,0 +1,97 @@ +# App Key Compromise Recovery + +This guide applies when an app key may have been disclosed. It covers recovery +from vulnerabilities in application code, the dstack OS, and KMS. For reporting +a newly discovered vulnerability, follow [SECURITY.md](../../SECURITY.md) before +sharing exploit details. + +## Recovery model + +KMS derives an app's key from the KMS root key and the app ID. Removing an +allowed compose hash, OS image, or device prevents future key release, but does +not revoke key material that has already left KMS. Reusing the same app ID under +the same KMS root also reproduces the same app identity. + +Consequently: + +- if an app key may be disclosed, retire that app ID; +- if the KMS root key may be disclosed, retire the entire KMS trust domain. + +dstack does not currently provide in-place app-key versioning or rotation. A +replacement identity is therefore the recovery boundary. + +## Immediate containment + +Before recovery: + +1. Stop or isolate affected app and KMS instances. +2. Remove affected compose hashes, OS image hashes, and device IDs from the + authorization policy. +3. Preserve attestation evidence, authorization history, logs, and relevant + on-chain records for investigation. +4. Identify every app ID for which a key could have been obtained. If the scope + cannot be established reliably, use the broader recovery case. + +Containment stops further key release. It does not restore trust in an existing +app ID or KMS root. + +## Choose the recovery boundary + +| Compromise | Required recovery | +| --- | --- | +| Application code discloses its app key | Create a new app ID for that application. The existing KMS domain can remain in use. | +| A dstack OS vulnerability discloses app keys | Remove the vulnerable OS image and create new app IDs for every affected application. If KMS ran on the affected OS, treat the KMS root as potentially disclosed. | +| A KMS authorization vulnerability releases app keys but cannot expose the KMS root | Fix KMS and create new app IDs for every application whose key could have been released. If the affected set is unknown, replace all app IDs in the domain. | +| KMS software, its OS, or its TEE may expose the KMS root | Create a new KMS trust domain and new app IDs for every application. | + +Arbitrary code execution inside KMS should normally be treated as potential +KMS-root disclosure: proving that an attacker did not copy a small root secret +is generally not possible. + +## Replace an app identity + +When the KMS root remains trusted: + +1. Fix and review the vulnerable application or OS. +2. Deploy a new app contract to obtain a new app ID. +3. Allow only the fixed compose hash and trusted OS image and devices. +4. Deploy the application with the new app ID. +5. Rotate application credentials and any business keys that the old instance + could access, including API tokens, database credentials, signing keys, and + data-encryption keys. +6. Update relying parties to trust the new app ID and reject the old identity. + +Treat data exported by a compromised instance as untrusted input. Validate it +before importing it, and re-encrypt retained confidential data under keys that +the old instance never possessed. + +## Replace a KMS trust domain + +When the KMS root may be disclosed, changing app IDs under that root is not a +recovery: an attacker with the root can derive their keys too. Instead: + +1. Fix the KMS, OS, TEE, or authorization vulnerability and revoke the affected + measurements. +2. Bootstrap a new KMS root in a trusted environment. Do not onboard or copy + root-key material from the affected KMS domain. +3. Publish the new KMS public-key trust anchor and update the KMS contract or + deployment configuration as applicable. +4. Create new app IDs for every application and rotate their credentials and + business keys. +5. Update gateways, verifiers, clients, and other relying parties to reject the + old KMS root and identities certified by it. + +## Completion checklist + +Recovery is complete only when: + +- vulnerable measurements and old app IDs can no longer obtain keys; +- replacement instances attest to the expected fixed software and policy; +- replacement app keys differ from the compromised identities; +- affected credentials and data-encryption keys have been rotated; +- all relying parties reject old app IDs and, when applicable, the old KMS + trust anchor. + +See [On-chain Governance](../onchain-governance.md) for authorization controls +and the [Security Model](./security-model.md) for the underlying trust +boundaries. diff --git a/docs/security/security-model.md b/docs/security/security-model.md index 751dbdc79..6e7427cea 100644 --- a/docs/security/security-model.md +++ b/docs/security/security-model.md @@ -326,12 +326,20 @@ kernel header for CoCo VMs", first released in 10.2.0) stopped rewriting the header for confidential guests, so the same kernel would otherwise measure differently depending on which QEMU the host chose to run. -dstack removes that dependency instead of modelling it. The image build zeroes -the boot-loader-written fields in the kernel it ships, and dstack's OVMF zeroes -them again before the kernel blob is measured and loaded. RTMR[1] is therefore -the plain Authenticode hash of the `bzImage` listed in `sha256sum.txt`, and the -verifier needs nothing from the host to predict it -- not a QEMU version, not a -memory size. +dstack removes that dependency instead of modelling it. The image build fills +in the boot-loader-owned fields of the kernel it ships with the values QEMU +<= 10.1 would write, and dstack's OVMF writes the same values again before the +kernel blob is measured and loaded. RTMR[1] is therefore the plain Authenticode +hash of the `bzImage` listed in `sha256sum.txt`, and the verifier needs nothing +from the host to predict it -- not a QEMU version, not a memory size. + +Normalizing to QEMU's layout rather than to zeros keeps an earlier release able +to verify these images: `dstack-mr` computes that layout for an image that does +not declare the flag, so a KMS that predates the declaration can still verify a +guest booted on a QEMU that no longer patches the header, and a root-key +handover does not have to bypass image verification. Recomputing it needs the +guest RAM size, which limits such a verifier to guests with exactly 2 GiB or at +least 2816 MiB. Images built before this landed keep their original behavior: their firmware does not normalize, so their digest still covers QEMU's rewritten copy. Which @@ -350,11 +358,14 @@ here there is no knob. It also removes a class of correct-but-rejected deployments, since the previous QEMU-patched digest varied with guest RAM and was only reproducible at specific memory sizes. -The normalized field set comes from the boot protocol rather than from QEMU's -behavior: every field `Documentation/arch/x86/boot.rst` types as `write` is one -the boot loader fills in and the kernel supplies no value for, so zeroing it -discards nothing the kernel provided. Fields typed `modify` carry real -kernel-supplied values and are left measured. +What is written is one fixed header -- the one QEMU <= 10.1 in fact wrote for +these kernels -- and not a reimplementation of QEMU's loader. Every field +written is one QEMU fills in as boot loader; all but one are typed `write` in +`Documentation/arch/x86/boot.rst`, which the boot loader supplies and the +kernel has no value for. The exception is `loadflags`, typed +`modify (obligatory)`, of which only the `CAN_USE_HEAP` bit is set, a bit the +protocol assigns to the boot loader. Fields that carry real kernel-supplied +values are left as the kernel built them. ### TCB status is surfaced, not gated, during verification @@ -376,6 +387,10 @@ This argument has a limit, and it is worth stating because it is what keeps the Production verifiers should reject deployments that use these development settings. Operators should treat them the same way they treat debug-mode TEE quotes: useful for testing, invalid for production trust. +### `requireTcbUpToDate` does not gate AWS NitroTPM + +NitroTPM attestations carry no TCB version or advisories, so the verifier reports `tcb_status = "UpToDate"` for every verified NitroTPM attestation, and `DstackApp.requireTcbUpToDate` always passes on it. `IAppAuth.AppBootInfo` has no `teeVariant`, so an app owner cannot exclude the platform on-chain; the only gate is the KMS-local `aws_nitro_tpm_key_release` (off by default). + ### KMS mTLS is route-enforced for sensitive operations The KMS Rocket TLS listener permits connections without a client certificate because some bootstrap and public metadata endpoints must be reachable before a client has an RA-TLS certificate. A certificate that is presented must carry an attestation, but the issuer that signed it is not checked and is not the authorization boundary for key material. diff --git a/docs/tdx-guest-scaling.md b/docs/tdx-guest-scaling.md new file mode 100644 index 000000000..879c7bd92 --- /dev/null +++ b/docs/tdx-guest-scaling.md @@ -0,0 +1,107 @@ +# TDX guest scaling knobs + +Many-vCPU TDX guests can run synchronization-heavy workloads (thread pools, +PyTorch CPU operators, `docker pull` extraction) several times slower than the +same guest without TDX. Most of the gap is wakeup overhead: every IPI and every +idle HLT costs a TD exit and a TD entry, which is several times more expensive +than an ordinary VM exit, and on large guests the cost grows further when many +vCPUs exit at the same time. + +The dstack guest kernel has three runtime switches for this. + +| Switch | Default | Where | Effect | +| --- | --- | --- | --- | +| `tdx_wake_q_batch` | on in TDX guests | `/sys/module/kernel/parameters/tdx_wake_q_batch` (also a boot parameter) | `wake_up_q()` (futex wake, condition variable broadcast, OpenMP pools) publishes wakeups immediately but sends the notification IPIs for up to 8 wakees as one mask IPI. Enabled at the end of boot. | +| `tdx_pv_single_ipi` | on in TDX guests | `/sys/module/kernel/parameters/tdx_pv_single_ipi` (also a boot parameter) | Single-target function IPIs use the KVM PV IPI hypercall instead of an emulated x2APIC ICR write. | +| `cpuidle_haltpoll` module | not loaded | `modprobe cpuidle_haltpoll force=1` / `rmmod cpuidle_haltpoll` | Idle vCPUs poll in the guest before halting. Wakeups that arrive during the window need neither an IPI nor a TD exit, at the cost of CPU spent polling. | + +The two kernel switches cannot be enabled outside a TDX guest, so ordinary VMs +are unaffected. Boot with `tdx_wake_q_batch=0 tdx_pv_single_ipi=0` +(or write `N` to the sysfs files) to get the previous behavior. + +The haltpoll governor parameters stay available under +`/sys/module/haltpoll/parameters/`. For TDX, a fixed window that does not +shrink works best (`guest_halt_poll_allow_shrink=N`, +`guest_halt_poll_grow_start` equal to `guest_halt_poll_ns`); the default +adaptive window tends to collapse to zero under slow wakeups and stay there. + +## Profiles + +| Profile | Settings | When | +| --- | --- | --- | +| default | kernel defaults: no guest polling, `tdx_wake_q_batch=Y`, `tdx_pv_single_ipi=Y` | all guests; no extra idle CPU | +| throughput | haltpoll module with a non-shrinking window, `tdx_wake_q_batch=Y`, `tdx_pv_single_ipi=N` | vCPUs are not overcommitted and throughput matters more than idle CPU | +| off | all switches off | reference behavior before these patches | + +Measured poll windows for `throughput`: 200 µs for a 32-vCPU guest and 4 ms +for a 248-vCPU guest. Both are roughly 20-25 times the TD exit round trip +observed at that vCPU concurrency; values in between are interpolations. + +Polling costs CPU while vCPUs idle-poll. Compared with an ordinary VM running +the same image, a `throughput` TD used about 0.15-0.2 extra host cores at idle +or at 200 requests per second, independent of vCPU count (4-32 vCPUs), and +about 0.9 cores more than `default` at 32 vCPUs with every vCPU 5% busy. The +`default` profile stayed within 0.02 cores of `off` in the same tests. + +## Helper script + +The image ships `/usr/lib/dstack/tdx-guest-tune.sh`, which applies one of the +profiles above, picks the poll window from the vCPU count, and prints the +resulting state: + +```bash +/usr/lib/dstack/tdx-guest-tune.sh throughput # or: default, off, status +/usr/lib/dstack/tdx-guest-tune.sh throughput --poll-us 1000 --pv-single Y +``` + +It refuses to run outside a TDX guest unless `--force` is given, and fails with +a message if the kernel lacks one of the switches. + +The script is experimental and may be removed from future images; the kernel +switches above are the stable interface. It is outside `PATH` for that reason. +When calling it from an app's `pre_launch_script`, guard the call so the app +still starts on images without the script (the pre-launch script runs under +`set -e`): + +```bash +t=/usr/lib/dstack/tdx-guest-tune.sh +if [ -x "$t" ]; then "$t" throughput || echo "tdx-guest-tune failed, continuing"; fi +``` + +## Boot time of large guests + +Two always-on guest kernel patches keep the boot time of many-vCPU TDs from +growing with the square of the vCPU count on hosts that map TD private memory +at 4K only (upstream TDX KVM, 7.0): + +- `0009` stops the page allocator from accepting 4 MiB of unaccepted memory on + every allocation before the zone watermarks are initialized. Before, SMP + bring-up of a 248-vCPU TD accepted about 13 GiB one 4K page at a time. +- `0010` converts the default SWIOTLB buffer to shared at `fs_initcall_sync` + instead of before SMP bring-up. Before, the host had to force every + still-spinning AP out of the TD for each of the buffer's 262144 pages. + +A 248-vCPU, 32 GiB TD on a Granite Rapids host with a 7.0 kernel reaches init in +about 40-50 seconds instead of 575-680 seconds. The SWIOTLB size is unchanged. + +## Observed results + +Emerald Rapids host, 32-vCPU guest, the same image as TD and as an ordinary VM, +switches changed at runtime on one TD boot, two alternating rounds (MiB/s; docker +in seconds): + +| Workload | Ordinary VM | TD `off` | TD `default` | +| --- | ---: | ---: | ---: | +| pool 1x32 | 2063 | 78 | 644 | +| PyTorch PASSIVE 1x32 | 2976 | 61 | 460 | +| pool 8x32 | 13137 | 2071 | 3852 | +| pool 4x8 | 37989 | 3612 | 12746 | +| PyTorch PASSIVE 8x32, 2 ms sleep | 784 | 478 | 656 | +| pool 1x4 | 11056 | 2129 | 5082 | +| `docker pull python:3.12`, local registry (s) | 29.6 | 48.0 | 43.5 | +| Geometric mean vs ordinary VM | 100% | 13.6% | 38.5% | + +`default` was not slower than `off` on any workload, and timer and request +wakeup latency stayed comparable. The `throughput` profile closes more of the +remaining gap on wakeup-heavy workloads in exchange for polling CPU. Results +depend on workload shape. diff --git a/docs/vmm-cli-user-guide.md b/docs/vmm-cli-user-guide.md index 6dddc2ba5..6f6e275bf 100644 --- a/docs/vmm-cli-user-guide.md +++ b/docs/vmm-cli-user-guide.md @@ -233,6 +233,7 @@ First, create an application composition file that describes your application: - `--public-sysinfo`: Make system info publicly accessible - `--env-file`: File with environment variables to encrypt - `--no-instance-id`: Disable unique instance identification +- `--no-storage-discard`: Keep deleted data-disk blocks instead of returning them to the host. Required by `deploy --disk-prealloc falloc|full`, and it also keeps filesystem allocation and deletion patterns from the host. The cost is that the image only ever grows: space freed inside the guest is never given back #### Example with Security Features @@ -268,6 +269,17 @@ Deploy your application with the compose file: - `--vcpu`: Number of virtual CPUs (default: 1) - `--memory`: Memory size (e.g., 1G, 512M, 2048M) - `--disk`: Disk size (e.g., 20G, 100G) +- `--disk-prealloc`: Preallocate the disk on the host: `off`, `metadata`, `falloc` or `full`. Omitting it uses the node's `cvm.disk_prealloc`, which ships as `off`. `falloc` reserves the full size up front, so a host that is out of space fails the VM's first start instead of leaving the guest to hit ENOSPC mid-run; `full` also zeroes it, which takes as long as writing the whole disk. The disk is created on first start, not at deploy time, so this is where the reservation -- and any failure to make it -- shows up. The VMM's `cvm.disk_prealloc` provides the default. + + `falloc` and `full` reserve host blocks and therefore require `storage_discard: false` in the compose file (`compose --no-storage-discard`), because a guest that may discard returns the reserved blocks to the host on its first trim. Deploying either with discard still enabled is rejected: + + ``` + disk preallocation (falloc) reserves host blocks, so app-compose must set + storage_discard = false; either turn discard off in the compose file, or redeploy + with disk_prealloc = "off" or "metadata" -- the mode is fixed at deployment + ``` + + Editing app-compose changes its hash and so the app id, which for a KMS-backed app means another on-chain whitelist entry. `metadata` reserves no data blocks, needs no compose change, and is left alone by this check. - `--port`: Port mappings (see Port Mapping section) - `--gpu`: GPU assignments - `--ppcie`: Enable PPCIE mode (attach ALL available GPUs and NVSwitches) diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 227ef2fae..e4156c67a 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -270,6 +270,17 @@ version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4858a9d740c5007a9069007c3b4e91152d0506f13c1b31dd49051fd537656156" +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -2110,6 +2121,7 @@ dependencies = [ "http-client", "k256", "load_config", + "moka", "parity-scale-codec", "ra-rpc", "ra-tls", @@ -2442,6 +2454,7 @@ dependencies = [ "tracing-subscriber", "url", "uuid", + "vfio-ioctls", "wait-timeout", "which 7.0.3", ] @@ -2668,6 +2681,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "ez-hash" version = "1.1.0" @@ -4513,10 +4546,13 @@ version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" dependencies = [ + "async-lock", "crossbeam-channel", "crossbeam-epoch", "crossbeam-utils", "equivalent", + "event-listener", + "futures-util", "parking_lot 0.12.5", "portable-atomic", "smallvec", @@ -5126,6 +5162,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.11.2" @@ -8329,12 +8371,52 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vfio-bindings" +version = "0.6.2" +source = "git+https://github.com/kvinwang/vfio.git?rev=1b560c2#1b560c20c8c294db0df5c6d8e0346ee7f9d89b62" + +[[package]] +name = "vfio-ioctls" +version = "0.8.0" +source = "git+https://github.com/kvinwang/vfio.git?rev=1b560c2#1b560c20c8c294db0df5c6d8e0346ee7f9d89b62" +dependencies = [ + "byteorder", + "libc", + "log", + "thiserror 2.0.18", + "vfio-bindings", + "vm-memory", + "vmm-sys-util", +] + [[package]] name = "virtue" version = "0.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" +[[package]] +name = "vm-memory" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b55e753c7725603745cb32b2287ef7ef3da05c03c7702cda3fa8abe25ae0465" +dependencies = [ + "libc", + "thiserror 2.0.18", + "winapi", +] + +[[package]] +name = "vmm-sys-util" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "506c62fdf617a5176827c2f9afbcf1be155b03a9b4bf9617a60dbc07e3a1642f" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "void" version = "1.0.2" diff --git a/dstack/Cargo.toml b/dstack/Cargo.toml index 6223e773e..61cc8e56f 100644 --- a/dstack/Cargo.toml +++ b/dstack/Cargo.toml @@ -126,6 +126,7 @@ dstack-verifier = { path = "verifier", default-features = false } size-parser = { path = "size-parser" } nvattest = { path = "nvattest" } wavekv = "2.1" +vfio-ioctls = { git = "https://github.com/kvinwang/vfio.git", rev = "1b560c2", default-features = false } # Core dependencies anyhow = { version = "1.0.97", default-features = false } diff --git a/dstack/build/shared/tests/test_release_namespaces.py b/dstack/build/shared/tests/test_release_namespaces.py new file mode 100644 index 000000000..c9732a7ae --- /dev/null +++ b/dstack/build/shared/tests/test_release_namespaces.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: © 2026 Alpha Compute +# SPDX-License-Identifier: Apache-2.0 + +"""Validate registry publish, verification and provenance agree for mixed-case forks.""" + +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[4] + + +class ReleaseNamespaceTests(unittest.TestCase): + """Execute owner normalization and resolve registry targets without publishing.""" + + def test_fork_release_targets_share_owner(self): + """Catch upstream registry paths that fork credentials cannot publish to.""" + workflows = ROOT / ".github/workflows" + for component in ("gateway", "kms", "verifier", "local-key-provider"): + with ( + self.subTest(component=component), + tempfile.TemporaryDirectory() as tmp, + ): + source = (workflows / f"{component}-release.yml").read_text() + workflow = yaml.safe_load(source) + steps = workflow["jobs"]["build-and-release"]["steps"] + normalize = next( + s for s in steps if s.get("name") == "Normalise registry owner" + ) + env_file = Path(tmp) / "env" + # Actions uses GNU Bash; macOS's bundled Bash predates lowercase expansion. + bash = shutil.which("bash") + subprocess.run( + [bash, "-e", "-c", normalize["run"]], + env={ + **os.environ, + "GITHUB_REPOSITORY_OWNER": "AlphaCompute", + "GITHUB_ENV": str(env_file), + }, + check=True, + ) + owner = dict( + line.split("=", 1) for line in env_file.read_text().splitlines() + )["OWNER_LC"] + self.assertEqual(owner, "alphacompute") + rendered = source.replace("${{ env.OWNER_LC }}", owner).replace( + "${OWNER_LC}", owner + ) + image = ( + "local-key-provider" + if component == "local-key-provider" + else f"dstack-{component}" + ) + target = f"ghcr.io/{owner}/{image}" + # These are separate external protocol surfaces, all of which must + # identify the artifact published by this repository's token. + parsed = yaml.safe_load(rendered) + resolved_steps = parsed["jobs"]["build-and-release"]["steps"] + provenance = next( + s + for s in resolved_steps + if s.get("name") == "Generate artifact attestation" + ) + self.assertEqual(provenance["with"]["subject-name"], target) + pull = next( + s + for s in resolved_steps + if s.get("name") == "Verify anonymous pull access" + )["run"] + self.assertIn(f"repository:{owner}/{image}:pull", pull) + self.assertIn(f"ghcr.io/v2/{owner}/{image}/manifests/", pull) + self.assertNotIn("ghcr.io/dstack-tee/", rendered) + self.assertNotIn("repository:dstack-tee/", rendered) + publish = next( + s + for s in resolved_steps + if "IMAGE_NAME" in s.get("env", {}) or "tags" in s.get("with", {}) + ) + published = publish.get("env", {}).get( + "IMAGE_NAME", publish.get("with", {}).get("tags") + ) + self.assertTrue(published.startswith(target + ":"), published) diff --git a/dstack/dstack-attest/src/attestation.rs b/dstack/dstack-attest/src/attestation.rs index e55d63827..b27367bc4 100644 --- a/dstack/dstack-attest/src/attestation.rs +++ b/dstack/dstack-attest/src/attestation.rs @@ -491,7 +491,14 @@ pub fn detect_tee_variant() -> Result { } bail!("Unsupported platform: GCP(-tdx)"); } - Platform::NitroEnclave => Ok(TeeVariant::DstackNitroEnclave), + Platform::NitroEnclave => { + // DMI is host-controlled; require the NSM device before selecting a + // variant that skips runtime measurement and MR_CONFIG_ID checks. + if std::path::Path::new("/dev/nsm").exists() { + return Ok(TeeVariant::DstackNitroEnclave); + } + bail!("unsupported platform: Nitro Enclave without /dev/nsm"); + } Platform::AwsEc2 => { if std::path::Path::new("/dev/tpmrm0").exists() || std::path::Path::new("/dev/tpm0").exists() @@ -1122,7 +1129,7 @@ impl AttestationV1 { // re-parse of the raw document, so the values that feed // os_image_hash / MR derivation are authenticated. let pcrs = NitroPcrs::from_verified(&verified_report.pcrs) - .context("verified NSM report missing PCR0/1/2")?; + .context("verified NSM report missing PCR0/1/2/4")?; DstackVerifiedReport::DstackNitroEnclave(NitroVerifiedReport { module_id: verified_report.module_id, pcrs, @@ -1211,6 +1218,9 @@ pub struct NitroPcrs { pub pcr1: Vec, #[serde(with = "serde_human_bytes")] pub pcr2: Vec, + /// PCR4 - Parent EC2 instance ID measurement + #[serde(with = "serde_human_bytes")] + pub pcr4: Vec, } impl NitroPcrs { @@ -1222,7 +1232,13 @@ impl NitroPcrs { let pcr0 = pcrs.get(&0).cloned().context("PCR 0 not found")?; let pcr1 = pcrs.get(&1).cloned().context("PCR 1 not found")?; let pcr2 = pcrs.get(&2).cloned().context("PCR 2 not found")?; - Ok(NitroPcrs { pcr0, pcr1, pcr2 }) + let pcr4 = pcrs.get(&4).cloned().context("PCR 4 not found")?; + Ok(NitroPcrs { + pcr0, + pcr1, + pcr2, + pcr4, + }) } fn is_zero(&self) -> bool { @@ -1266,7 +1282,13 @@ impl DstackNitroQuote { let pcr0 = doc.pcrs.get(&0).cloned().context("PCR 0 not found")?; let pcr1 = doc.pcrs.get(&1).cloned().context("PCR 1 not found")?; let pcr2 = doc.pcrs.get(&2).cloned().context("PCR 2 not found")?; - Ok(NitroPcrs { pcr0, pcr1, pcr2 }) + let pcr4 = doc.pcrs.get(&4).cloned().context("PCR 4 not found")?; + Ok(NitroPcrs { + pcr0, + pcr1, + pcr2, + pcr4, + }) } } @@ -1580,14 +1602,7 @@ impl GetDeviceId for DstackVerifiedReport { DstackVerifiedReport::DstackTdx(tdx_report) => tdx_report.ppid.to_vec(), DstackVerifiedReport::DstackAmdSevSnp(report) => report.chip_id.to_vec(), DstackVerifiedReport::DstackGcpTdx { tdx_report, .. } => tdx_report.ppid.to_vec(), - DstackVerifiedReport::DstackNitroEnclave(report) => { - // i-1234567890abcdef0-enc9876543210abcde -> i-1234567890abcdef0 - report - .module_id - .split_once('-') - .map(|(id, _)| id.as_bytes().to_vec()) - .unwrap_or_default() - } + DstackVerifiedReport::DstackNitroEnclave(report) => report.pcrs.pcr4.clone(), DstackVerifiedReport::DstackAwsNitroTpm(report) => report.module_id.as_bytes().to_vec(), } } @@ -1657,7 +1672,7 @@ fn decode_app_info_sev_snp( embedded_config: &str, external_vm_config: &str, ) -> Result { - let parsed = crate::amd_sev_snp::parse_amd_snp_report(report)?; + let parsed = crate::amd_sev_snp::parse_unverified_amd_snp_report(report)?; let mr_config_document = if let Some(mr_config) = mr_config { Cow::Borrowed(mr_config) } else if let Some(mr_config) = mr_config_document_from_config(external_vm_config)? { @@ -2472,10 +2487,8 @@ impl Attestation { bail!("NSM user_data does not match report_data"); } - // Decode PCRs from quote - let pcrs = nsm_quote - .decode_pcrs() - .context("Failed to decode nitro pcrs")?; + let pcrs = NitroPcrs::from_verified(&verified_report.pcrs) + .context("verified NSM report missing PCR0/1/2/4")?; Ok(NitroVerifiedReport { module_id: verified_report.module_id, @@ -3052,20 +3065,25 @@ mod tests { } #[test] - fn nitro_pcrs_from_verified_extracts_0_1_2() { + fn nitro_pcrs_from_verified_extracts_0_1_2_4() { let mut map = std::collections::BTreeMap::new(); map.insert(0u16, vec![0xaa; 48]); map.insert(1u16, vec![0xbb; 48]); map.insert(2u16, vec![0xcc; 48]); map.insert(3u16, vec![0xdd; 48]); // ignored + map.insert(4u16, vec![0xee; 48]); let pcrs = NitroPcrs::from_verified(&map).unwrap(); assert_eq!(pcrs.pcr0, vec![0xaa; 48]); assert_eq!(pcrs.pcr1, vec![0xbb; 48]); assert_eq!(pcrs.pcr2, vec![0xcc; 48]); + assert_eq!(pcrs.pcr4, vec![0xee; 48]); // missing a required PCR is an error map.remove(&1u16); assert!(NitroPcrs::from_verified(&map).is_err()); + map.insert(1u16, vec![0xbb; 48]); + map.remove(&4u16); + assert!(NitroPcrs::from_verified(&map).is_err()); } #[test] @@ -3074,6 +3092,7 @@ mod tests { pcr0: vec![0u8; 48], pcr1: vec![0u8; 48], pcr2: vec![0u8; 48], + pcr4: vec![0u8; 48], }; assert!(debug.is_debug()); @@ -3081,6 +3100,7 @@ mod tests { pcr0: vec![1u8; 48], pcr1: vec![0u8; 48], pcr2: vec![0u8; 48], + pcr4: vec![0u8; 48], }; assert!(!prod.is_debug()); // image_hash = sha256(pcr0 || pcr1 || pcr2), never the all-zero sentinel @@ -3295,4 +3315,25 @@ mod tests { .is_err() ); } + + fn nitro_report_with_pcr4(pcr4: Vec) -> DstackVerifiedReport { + DstackVerifiedReport::DstackNitroEnclave(NitroVerifiedReport { + module_id: "opaque-module-id".to_string(), + pcrs: NitroPcrs { + pcr0: vec![0x11; 48], + pcr1: vec![0x22; 48], + pcr2: vec![0x33; 48], + pcr4, + }, + user_data: vec![], + timestamp: 0, + }) + } + + #[test] + fn nitro_enclave_device_id_is_parent_instance_pcr4() { + let pcr4 = vec![0x44; 48]; + let report = nitro_report_with_pcr4(pcr4.clone()); + assert_eq!(report.get_devide_id(), pcr4); + } } diff --git a/dstack/dstack-attest/src/v1.rs b/dstack/dstack-attest/src/v1.rs index f895a94cf..168fd58a8 100644 --- a/dstack/dstack-attest/src/v1.rs +++ b/dstack/dstack-attest/src/v1.rs @@ -7,7 +7,6 @@ use cc_eventlog::{ tdx::{self, TDX_ACPI_DATA_EVENT_PAYLOAD}, RuntimeEvent, TdxEvent, }; -use dstack_types::mr_config::MrConfigV3; use serde::{Deserialize, Serialize}; use tpm_types::TpmQuote; @@ -155,11 +154,6 @@ impl PlatformEvidence { } } - pub fn sev_snp_mr_config(&self) -> Option { - self.sev_snp_mr_config_document() - .and_then(|document| MrConfigV3::from_document(document).ok()) - } - pub fn tdx_event_log_mut(&mut self) -> Option<&mut Vec> { match self { Self::Tdx { event_log, .. } | Self::GcpTdx { event_log, .. } => Some(event_log), diff --git a/dstack/dstack-attest/tests/nitro_verify.rs b/dstack/dstack-attest/tests/nitro_verify.rs index 5becbb36e..2c8f615c6 100644 --- a/dstack/dstack-attest/tests/nitro_verify.rs +++ b/dstack/dstack-attest/tests/nitro_verify.rs @@ -5,7 +5,7 @@ //! Integration test: verify Nitro Enclave attestation end-to-end use dstack_attest::attestation::{ - AttestationQuote, AttestationVerifier, DstackVerifiedReport, VersionedAttestation, + AttestationQuote, AttestationVerifier, DstackVerifiedReport, GetDeviceId, VersionedAttestation, }; use nsm_qvl::{AttestationDocument, CoseSign1}; use std::time::{Duration, SystemTime}; @@ -47,6 +47,16 @@ async fn verify_nitro_attestation_bin() { .verify_with_time(&verifier, Some(fixed_now)) .await .unwrap(); + // PCR4 is the AWS-defined measurement of the parent EC2 instance ID. + assert_eq!( + verified.report.get_devide_id(), + hex::decode( + "7b3574ad9b7e88ec8335e8ce967315fac77218441e97eea9795de704d7f2678d\ + 0b298ecda39b544c1689669f48cba5e8" + ) + .unwrap() + ); + let DstackVerifiedReport::DstackNitroEnclave(report) = verified.report else { panic!("Nitro attestation verification failed"); }; diff --git a/dstack/dstack-attest/tests/sev_snp_verify.rs b/dstack/dstack-attest/tests/sev_snp_verify.rs index 38a19145c..73455dc74 100644 --- a/dstack/dstack-attest/tests/sev_snp_verify.rs +++ b/dstack/dstack-attest/tests/sev_snp_verify.rs @@ -406,3 +406,44 @@ fn advertised_os_image_hash_must_match_sha256sum() { "unexpected error: {err:?}" ); } + +/// The whole `POST /verify` path, with the SNP report emptied. +/// +/// `dstack-verifier` decodes the request body with +/// `VersionedAttestation::from_bytes` and hands the result straight to +/// `verify`, which routes an SNP quote with no `cert_chain` to the AMD KDS +/// branch. That branch used to parse the report without checking its length, +/// and release builds are `panic = "abort"`, so this body took the verifier +/// process down rather than failing one request. +#[tokio::test] +async fn an_empty_snp_report_from_the_verify_body_is_rejected_rather_than_parsed() { + let VersionedAttestation::V0 { mut attestation } = + VersionedAttestation::from_scale(SEV_ATTESTATION_BIN).expect("decode VersionedAttestation") + else { + panic!("expected V0 attestation"); + }; + let AttestationQuote::DstackAmdSevSnp(quote) = &mut attestation.quote else { + panic!("expected an AMD SEV-SNP quote"); + }; + quote.report = Vec::new(); + quote.cert_chain = Vec::new(); + + let body = attestation + .into_versioned() + .to_bytes() + .expect("encode attestation"); + let verifier = dstack_attest::attestation::AttestationVerifier::new_prod(None) + .expect("build a production verifier"); + let Err(err) = VersionedAttestation::from_bytes(&body) + .expect("decode attestation") + .into_v1() + .verify(&verifier) + .await + else { + panic!("an empty SNP report must be rejected"); + }; + assert!( + format!("{err:#}").contains("invalid amd sev-snp report length"), + "unexpected error: {err:#}" + ); +} diff --git a/dstack/dstack-attest/tests/snapshots/nitro_verify__nitro_report.snap b/dstack/dstack-attest/tests/snapshots/nitro_verify__nitro_report.snap index dafa10583..855d2f834 100644 --- a/dstack/dstack-attest/tests/snapshots/nitro_verify__nitro_report.snap +++ b/dstack/dstack-attest/tests/snapshots/nitro_verify__nitro_report.snap @@ -8,7 +8,8 @@ expression: "serde_json::to_string_pretty(&report).unwrap()" "pcrs": { "pcr0": "eb7d0dab08ff41546d7e5659aee883af7b32bb2e58e46815b719f9f7bfae7b880188a10d86317e6509923740526cf74a", "pcr1": "0343b056cd8485ca7890ddd833476d78460aed2aa161548e4e26bedf321726696257d623e8805f3f605946b3d8b0c6aa", - "pcr2": "d7b0a76788c1be24898bd148117ea1239578ba0e72f5d87a33a6c6476026675b6f996940f062e0e3f0b5edd2ddf78811" + "pcr2": "d7b0a76788c1be24898bd148117ea1239578ba0e72f5d87a33a6c6476026675b6f996940f062e0e3f0b5edd2ddf78811", + "pcr4": "7b3574ad9b7e88ec8335e8ce967315fac77218441e97eea9795de704d7f2678d0b298ecda39b544c1689669f48cba5e8" }, "user_data": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8550000000000000000000000000000000000000000000000000000000000000000", "timestamp": 1766678659346 diff --git a/dstack/dstack-mr/src/kernel.rs b/dstack/dstack-mr/src/kernel.rs index 7a130321d..38977feee 100644 --- a/dstack/dstack-mr/src/kernel.rs +++ b/dstack/dstack-mr/src/kernel.rs @@ -173,7 +173,10 @@ fn patch_kernel( let mut initrd_max = if protocol >= 0x20c { let xlf = u16::from_le_bytes(kd[0x236..0x238].try_into().context("impossible failure")?); - if (xlf & 0x40) != 0 { + // XLF_CAN_BE_LOADED_ABOVE_4G is bit 1. Bit 6, 0x40, is + // XLF_5LEVEL_ENABLED; a kernel built without 5-level paging sets + // one and not the other, and QEMU tests bit 1. + if (xlf & 0x02) != 0 { u32::MAX } else { 0x37ffffff @@ -240,8 +243,8 @@ pub(crate) fn patched_kernel_authenticode_sha384( /// Compute the first RTMR[1] event digest for an image whose OVMF normalizes /// the Linux setup header: the Authenticode SHA-384 of the kernel file itself. /// -/// Both sides zero the boot-loader-written fields -- -/// `os/image/normalize-kernel-header.py` in the image build and +/// Both sides fill in the boot-loader-owned fields with the values QEMU <= 10.1 +/// writes -- `os/image/normalize-kernel-header.py` in the image build and /// `0007-OvmfPkg-QemuKernelLoaderFsDxe-normalize-setup-header.patch` in the /// firmware -- so what OVMF measures is the file, on every QEMU version and at /// every guest memory size. @@ -308,7 +311,7 @@ mod tests { kernel[0x206..0x208].copy_from_slice(&0x020cu16.to_le_bytes()); // XLF_CAN_BE_LOADED_ABOVE_4G, so QEMU derives the initrd address from // available low memory and the patched digest moves with guest RAM. - kernel[0x236..0x238].copy_from_slice(&0x0040u16.to_le_bytes()); + kernel[0x236..0x238].copy_from_slice(&0x0002u16.to_le_bytes()); kernel[0x224..0x226].copy_from_slice(&0x50a0u16.to_le_bytes()); kernel } @@ -340,13 +343,37 @@ mod tests { assert_ne!(patched_at(0x8000_0000), patched_at(0xA000_0000)); } + /// QEMU derives the initrd ceiling from XLF_CAN_BE_LOADED_ABOVE_4G, + /// which is bit 1; bit 6 is XLF_5LEVEL_ENABLED and must not be read as it. + #[test] + fn only_xlf_bit_1_raises_the_initrd_ceiling() { + let mut kernel = vec![0u8; 0x1000]; + kernel[0x206..0x208].copy_from_slice(&0x020cu16.to_le_bytes()); + + // The initrd the 0.6.0 image ships, so these are the addresses QEMU + // puts it at and the image build writes into the shipped kernel. + let initrd_size = 6_454_798; + let with_xlf = |xlf: u16| { + let mut k = kernel.clone(); + k[0x236..0x238].copy_from_slice(&xlf.to_le_bytes()); + initrd_addr(&patch_kernel(&k, initrd_size, 0x8000_0000, 0x28000).unwrap()) + }; + + // 0x37ffffff is the ceiling for a kernel that cannot be loaded above + // 4G; the raised ceiling is the below-4G window minus the ACPI area. + assert_eq!(with_xlf(0x0000), 0x379D_8000); + assert_eq!(with_xlf(0x0040), 0x379D_8000, "the 5-level flag raised it"); + assert_eq!(with_xlf(0x0002), 0x7F9B_0000, "the above-4G flag did not"); + assert_eq!(with_xlf(0x0042), 0x7F9B_0000); + } + #[test] fn tdx_kernel_patch_uses_precomputed_digest_at_2g_and_high_memory() { let mut kernel = vec![0u8; 0x1000]; // Linux boot protocol >= 2.12 with XLF_CAN_BE_LOADED_ABOVE_4G makes // QEMU derive the initrd address from available low memory. kernel[0x206..0x208].copy_from_slice(&0x020cu16.to_le_bytes()); - kernel[0x236..0x238].copy_from_slice(&0x0040u16.to_le_bytes()); + kernel[0x236..0x238].copy_from_slice(&0x0002u16.to_le_bytes()); let below_2g = patch_kernel(&kernel, 0x100000, 0x80000000 - 0x1000, 0x28000).unwrap(); let at_2g = patch_kernel(&kernel, 0x100000, 0x80000000, 0x28000).unwrap(); diff --git a/dstack/dstack-mr/src/main.rs b/dstack/dstack-mr/src/main.rs index 7b5e52d53..0b71c00dd 100644 --- a/dstack/dstack-mr/src/main.rs +++ b/dstack/dstack-mr/src/main.rs @@ -134,11 +134,8 @@ fn inspect_measurement(kind: &str, path: &Path) -> Result { .map_err(anyhow::Error::msg), "gcp" => dstack_types::GcpOsImageMeasurement::cbor_json_value_from_slice(&cbor) .map_err(anyhow::Error::msg), - "aws" => { - let measurement = dstack_types::AwsOsImageMeasurement::from_cbor_slice(&cbor) - .map_err(anyhow::Error::msg)?; - serde_json::to_value(measurement).context("failed to convert AWS measurement to JSON") - } + "aws" => dstack_types::AwsOsImageMeasurement::cbor_json_value_from_slice(&cbor) + .map_err(anyhow::Error::msg), other => bail!("unknown measurement kind {other:?}; expected tdx, snp, gcp, or aws"), } } diff --git a/dstack/dstack-mr/src/sev.rs b/dstack/dstack-mr/src/sev.rs index b1d419a49..172fcc1a4 100644 --- a/dstack/dstack-mr/src/sev.rs +++ b/dstack/dstack-mr/src/sev.rs @@ -34,6 +34,37 @@ pub const MAX_OVMF_METADATA_PAGES: u64 = 16_777_216; // VMSA page GPA: (u64)(-1) page-aligned, bits >51 cleared. const VMSA_GPA: u64 = 0x0000_FFFF_FFFF_F000; +/// `SNPActive`, bit 0 of `SEV_FEATURES` (AMD64 APM vol. 2, VMCB SEV_FEATURES; +/// the same bit layout the SEV-SNP ABI's `GUEST_FEATURES` field in +/// `SNP_LAUNCH_START` carries). Set on every SNP guest. +const SEV_FEATURE_SNP_ACTIVE: u64 = 1 << 0; + +/// `SEV_FEATURES` bits a dstack launch may carry. +/// +/// `guest_features` is the only launch parameter that is neither pinned by +/// `SevOsImageMeasurement` nor otherwise constrained: it rides in the +/// host-written `SnpMeasurementDocument` alongside `vcpus`/`vcpu_type`, outside +/// the CBOR that `os_image_hash` commits to, and lands verbatim in both VMSA +/// pages at offset 0x3B0. Because the expected launch digest is *recomputed* +/// from the declared value, a host that really booted the guest with extra +/// feature bits and declares them gets a matching digest and a key release — +/// the bits themselves were never policy-checked. The dangerous direction is +/// DebugSwap (bit 5), which swaps the guest's debug registers on VMEXIT and so +/// exposes guest state to the hypervisor; clearing RestrictedInjection (bit 3), +/// SecureTSC (bit 9) or VmsaRegProt (bit 14) likewise removes a guest-side +/// protection the operator may believe is on. +/// +/// dstack has exactly one launch path, and it sets no feature property: +/// `sev-snp-guest,id=sev0,policy=0x30000,...` in `dstack/vmm/src/app/qemu.rs`, +/// which leaves KVM to start the guest with `SNPActive` alone, and the VMM +/// writes the matching `guest_features: 1` into the measurement document +/// (`dstack/vmm/src/app.rs`). `1` is therefore the only value dstack has ever +/// produced — the captured real-hardware vector in this module's tests carries +/// it too — so it is the only value accepted. Widening this is a one-line +/// change once a launch path actually sets another bit; accepting bits nothing +/// emits would only ever admit a guest dstack did not configure. +const ALLOWED_GUEST_FEATURES: u64 = SEV_FEATURE_SNP_ACTIVE; + #[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize)] #[serde(deny_unknown_fields)] pub struct OvmfSectionParam { @@ -111,8 +142,16 @@ where /// Validate a `MeasurementInput` for shape/bounds before recomputation. pub fn validate_measurement_input(input: &MeasurementInput) -> Result<()> { - if input.guest_features == 0 { - bail!("guest_features must be non-zero"); + if input.guest_features & SEV_FEATURE_SNP_ACTIVE == 0 { + bail!("guest_features must set SNPActive (bit 0)"); + } + let unexpected_features = input.guest_features & !ALLOWED_GUEST_FEATURES; + if unexpected_features != 0 { + bail!( + "guest_features {:#x} carries unsupported SEV_FEATURES bits {unexpected_features:#x}; \ + dstack launches amd sev-snp guests with SNPActive only ({ALLOWED_GUEST_FEATURES:#x})", + input.guest_features + ); } rootfs_hash_from_cmdline(Some(&input.base_cmdline))?; @@ -836,16 +875,33 @@ fn file_sha256(path: &Path) -> Result> { Ok(Sha256::digest(data).to_vec()) } +/// Read the rootfs identity the measured kernel command line commits to. +/// +/// A duplicated `dstack.rootfs_hash=` is rejected rather than resolved. The +/// Linux command line has no "first wins" rule -- the dstack initramfs that +/// mounts the rootfs reads the last occurrence -- so returning the first, as +/// this did, would have answered with an identity the guest did not use. Taking +/// the last would agree with the initramfs but would still accept a measured +/// command line asserting two different rootfs identities, which is not a shape +/// any dstack image produces and not one a reader can disambiguate. Callers use +/// this as a validation gate on a command line an untrusted host supplies, so +/// it fails closed. pub fn rootfs_hash_from_cmdline(cmdline: Option<&str>) -> Result { - let rootfs_hash = cmdline - .unwrap_or_default() - .split_whitespace() - .find_map(|param| param.strip_prefix("dstack.rootfs_hash=")) - .map(ToString::to_string) - .context("dstack.rootfs_hash is required in amd sev-snp measured cmdline")?; + let mut rootfs_hash = None; + for param in cmdline.unwrap_or_default().split_whitespace() { + let Some(value) = param.strip_prefix("dstack.rootfs_hash=") else { + continue; + }; + if rootfs_hash.is_some() { + bail!("dstack.rootfs_hash appears more than once in the measured cmdline"); + } + rootfs_hash = Some(value); + } + let rootfs_hash = + rootfs_hash.context("dstack.rootfs_hash is required in amd sev-snp measured cmdline")?; Ok(hex::encode(decode_required_hex( "dstack.rootfs_hash", - &rootfs_hash, + rootfs_hash, 32, )?)) } @@ -1527,6 +1583,103 @@ mod tests { assert_eq!(binding.mr_config.app_id, mr_config.app_id); } + /// A host that really boots the guest with DebugSwap and declares it is not + /// tampering: every field is internally consistent, the recomputed launch + /// digest matches the hardware `MEASUREMENT`, and `host_data` matches the + /// mr_config. The only thing standing between that guest and a key release + /// is a policy check on `guest_features` itself. + /// Bit positions from the AMD64 APM `SEV_FEATURES` table, which the SEV-SNP + /// ABI `GUEST_FEATURES` field in `SNP_LAUNCH_START` mirrors. + const SNP_ACTIVE: u64 = 1 << 0; + const DEBUG_SWAP: u64 = 1 << 5; + + /// `find_map` took the first `dstack.rootfs_hash=`, while the initramfs + /// that actually mounts the rootfs honours the last one. The function is + /// `pub` and reads authoritative, so a caller that trusted it would have + /// been told a different rootfs identity than the guest used. + #[test] + fn a_duplicated_rootfs_hash_is_not_silently_resolved() { + let first = hex_of(0x11, 32); + let last = hex_of(0x22, 32); + let cmdline = format!( + "console=ttyS0 dstack.rootfs_hash={first} init=/init dstack.rootfs_hash={last}" + ); + let err = match rootfs_hash_from_cmdline(Some(&cmdline)) { + Ok(hash) => panic!("a duplicated rootfs hash resolved to {hash}"), + Err(err) => err.to_string(), + }; + assert!( + err.contains("dstack.rootfs_hash appears more than once"), + "unexpected error: {err}" + ); + + // A single occurrence is unchanged. + let cmdline = format!("console=ttyS0 dstack.rootfs_hash={first}"); + assert_eq!( + rootfs_hash_from_cmdline(Some(&cmdline)).expect("single occurrence"), + first + ); + } + + #[test] + fn verify_sev_launch_rejects_a_consistent_debugswap_guest() { + let mut input = valid_input(); + input.guest_features = SNP_ACTIVE | DEBUG_SWAP; + let mr_config = synthetic_mr_config(); + let host_data = MrConfigV3::snp_host_data_from_document(&mr_config.to_canonical_json()); + let measurement = compute_expected_measurement(&input).expect("measurement"); + let vm_config = synthetic_vm_config(&input, &mr_config); + + let err = match verify_sev_launch(&measurement, &host_data, &vm_config) { + Ok(binding) => panic!( + "a DebugSwap launch verified; bound os_image_hash {}", + hex::encode(binding.os_image_hash) + ), + Err(err) => err.to_string(), + }; + assert!( + err.contains("unsupported SEV_FEATURES bits 0x20"), + "unexpected error: {err}" + ); + } + + #[test] + fn rejects_guest_features_outside_the_launch_allowlist() { + let cases: [(&str, u64); 4] = [ + ("RestrictedInjection", SNP_ACTIVE | (1 << 3)), + ("DebugSwap", SNP_ACTIVE | (1 << 5)), + ("SecureTSC", SNP_ACTIVE | (1 << 9)), + ("reserved bit 63", SNP_ACTIVE | (1 << 63)), + ]; + for (name, features) in cases { + let mut input = valid_input(); + input.guest_features = features; + let err = validate_measurement_input(&input) + .expect_err("feature bit outside the allowlist must not be accepted") + .to_string(); + assert!( + err.contains("unsupported SEV_FEATURES bits"), + "{name}: unexpected error: {err}" + ); + } + + // SNPActive missing is still rejected, now by name. + let mut input = valid_input(); + input.guest_features = 0; + let err = validate_measurement_input(&input) + .expect_err("guest_features 0 must not be accepted") + .to_string(); + assert!( + err.contains("must set SNPActive"), + "unexpected error: {err}" + ); + + // The one value dstack's launch path produces stays accepted. + let mut input = valid_input(); + input.guest_features = SNP_ACTIVE; + validate_measurement_input(&input).expect("SNPActive-only launch is accepted"); + } + #[test] fn verify_sev_launch_rejects_forged_measurement() { let (_input, _mr, measurement, host_data, vm_config) = honest_case(); @@ -1579,7 +1732,12 @@ mod tests { ("vcpu_type", |i| { i.vcpu_type = Some("epyc-milan".to_string()) }), - ("guest_features", |i| i.guest_features = 3), + // guest_features is not in this list: the SEV_FEATURES allowlist + // now rejects every value but SNPActive before the measurement is + // ever recomputed, so there is no tampered value left to reach the + // measurement comparison. See + // `rejects_guest_features_outside_the_launch_allowlist` and + // `verify_sev_launch_rejects_a_consistent_debugswap_guest`. ]; for (name, mutate) in cases { let mut tampered = input.clone(); diff --git a/dstack/dstack-mr/src/tdvf.rs b/dstack/dstack-mr/src/tdvf.rs index 3b6b7d3af..3f6393713 100644 --- a/dstack/dstack-mr/src/tdvf.rs +++ b/dstack/dstack-mr/src/tdvf.rs @@ -20,6 +20,23 @@ const ATTRIBUTE_PAGE_AUG: u32 = 0x00000002; const TDVF_SECTION_TD_HOB: u32 = 0x02; const TDVF_SECTION_TEMP_MEM: u32 = 0x03; +/// Largest section count accepted from a firmware image. Real TDVF metadata +/// describes six sections -- BFV, CFV, TD_HOB and three TEMP_MEM ranges -- so +/// this is an order of magnitude of headroom, and it matches the limit the +/// OVMF/SEV metadata parser already applies (`sev::MAX_OVMF_SECTIONS`). +pub(crate) const MAX_TDVF_SECTIONS: usize = 64; + +/// Size of each `TdvfSection` record in the section table. +const TDVF_SECTION_SIZE: usize = 32; + +/// Largest number of guest pages the whole section table may ask to be +/// measured. Real TDVF metadata measures 0x41a000 bytes across its six +/// sections -- 1050 pages, 4.10 MiB -- so 256 MiB of pages is about sixty +/// times the headroom any firmware needs. Without it a page-aligned +/// `memory_data_size` of 0x1000_0000_0000 is 2^36 SHA-384 updates, with +/// nothing between the file and the hash loop to stop it. +const MAX_MEASURED_PAGES: u64 = 0x1_0000; + pub enum PageAddOrder { TwoPass, SinglePass, @@ -177,12 +194,20 @@ impl<'a> Tdvf<'a> { let mut data: Option<&[u8]> = None; let encoded_guid = encode_guid(TDX_METADATA_OFFSET_GUID)?; loop { - if offset < 18 { + // `tables` starts 18 bytes before the real table, which is exactly + // the footer entry the walk has already consumed, so an offset of + // 18 means every entry has been visited. Reading one more "entry" + // there would interpret whatever precedes the table as a length. + if offset <= 18 { break; } let guid = &tables[offset - 16..offset]; let entry_len = read_le::(tables, offset - 18, "entry length")? as usize; - if entry_len > offset.saturating_sub(18) { + // An entry's length covers its own 18-byte length+GUID header, so + // anything shorter is malformed -- and a zero-length entry leaves + // `offset` where it is, walking the table forever. The OVMF/SEV + // footer walk rejects the same shape in `ovmf_footer_entries`. + if entry_len < 18 || entry_len > offset.saturating_sub(18) { bail!("Failed to parse TDVF metadata: Invalid entry length"); } if guid == encoded_guid { @@ -216,6 +241,9 @@ impl<'a> Tdvf<'a> { bail!("Failed to parse TDVF metadata: Unsupported TDVF version"); } let num_sections = descriptor.num_sections as usize; + if num_sections > MAX_TDVF_SECTIONS { + bail!("TDVF metadata declares {num_sections} sections, more than the {MAX_TDVF_SECTIONS} supported"); + } let mut meta = Tdvf { fw, @@ -223,9 +251,23 @@ impl<'a> Tdvf<'a> { }; // Decode all sections using scale codec + let mut total_pages = 0u64; for i in 0..num_sections { - let sec_offset = tdvf_meta_offset + 16 + 32 * i; - let s = TdvfSection::decode(&mut &fw[sec_offset..]) + // `num_sections` is a raw `u32` from the file, so the table it + // describes need not be inside the image. Slice with `get` rather + // than indexing: release builds abort on panic, so an out-of-range + // start would take down the whole measuring process instead of + // failing this one image. + let sec_offset = tdvf_meta_offset + 16 + TDVF_SECTION_SIZE * i; + let record = fw + .get(sec_offset..sec_offset + TDVF_SECTION_SIZE) + .with_context(|| { + format!( + "TDVF section {i} at offset {sec_offset} is outside the {}-byte firmware", + fw.len() + ) + })?; + let s = TdvfSection::decode(&mut &record[..]) .with_context(|| format!("failed to decode TDVF section {}", i))?; if s.memory_address % PAGE_SIZE != 0 { @@ -242,6 +284,43 @@ impl<'a> Tdvf<'a> { { bail!("Failed to parse TDVF metadata: Section raw data size less than memory"); } + // MR.EXTEND hashes one page of firmware bytes per measured page, + // so it reads `memory_data_size` bytes from `data_offset` -- not + // `raw_data_size`. Nothing above ties either to the image, and the + // read is what panics, so check the declared range here, before + // any of it is measured. + let measured_size = if s.attributes & ATTRIBUTE_MR_EXTEND != 0 { + s.memory_data_size + } else { + s.raw_data_size as u64 + }; + let data_end = (s.data_offset as u64) + .checked_add(measured_size) + .with_context(|| format!("TDVF section {i} data range overflows"))?; + if data_end > fw.len() as u64 { + bail!( + "TDVF section {i} data range {}..{data_end} is outside the {}-byte firmware", + s.data_offset, + fw.len() + ); + } + // The guest address of a measured page is `memory_address + page * + // PAGE_SIZE`, and both halves come from the file. In a release + // build that addition wraps silently and measures the wrong + // addresses; in a debug build it panics. + if s.memory_address.checked_add(s.memory_data_size).is_none() { + bail!( + "TDVF section {i} guest address range wraps past the end of the address space" + ); + } + // `memory_data_size` alone decides how many pages get hashed, so + // an unconstrained one is an unbounded amount of work. + total_pages = total_pages + .checked_add(s.memory_data_size / PAGE_SIZE) + .with_context(|| format!("TDVF section {i} page count overflows"))?; + if total_pages > MAX_MEASURED_PAGES { + bail!("TDVF metadata asks to measure {total_pages} pages, more than the {MAX_MEASURED_PAGES} supported"); + } meta.sections.push(s); } @@ -252,32 +331,44 @@ impl<'a> Tdvf<'a> { fn compute_mrtd(&self, variant: PageAddOrder) -> Result> { let mut h = Sha384::new(); - let mem_page_add = |h: &mut Sha384, s: &TdvfSection, page: u64| { + let mem_page_add = |h: &mut Sha384, s: &TdvfSection, page: u64| -> Result<()> { if s.attributes & ATTRIBUTE_PAGE_AUG == 0 { let mut buf = [0u8; 128]; buf[..12].copy_from_slice(b"MEM.PAGE.ADD"); - let gpa = s.memory_address + page * PAGE_SIZE; + let gpa = page + .checked_mul(PAGE_SIZE) + .and_then(|offset| s.memory_address.checked_add(offset)) + .context("TDVF section guest address wraps")?; buf[16..24].copy_from_slice(&gpa.to_le_bytes()); h.update(buf); } + Ok(()) }; - let mr_extend = |h: &mut Sha384, s: &TdvfSection, page: u64| { + let mr_extend = |h: &mut Sha384, s: &TdvfSection, page: u64| -> Result<()> { if s.attributes & ATTRIBUTE_MR_EXTEND != 0 { for i in 0..(PAGE_SIZE as usize / MR_EXTEND_GRANULARITY) { let mut buf = [0u8; 128]; buf[..9].copy_from_slice(b"MR.EXTEND"); - let gpa = - s.memory_address + page * PAGE_SIZE + (i * MR_EXTEND_GRANULARITY) as u64; + let gpa = page + .checked_mul(PAGE_SIZE) + .and_then(|offset| offset.checked_add((i * MR_EXTEND_GRANULARITY) as u64)) + .and_then(|offset| s.memory_address.checked_add(offset)) + .context("TDVF section guest address wraps")?; buf[16..24].copy_from_slice(&gpa.to_le_bytes()); h.update(buf); let chunk_offset = s.data_offset as usize + (page * PAGE_SIZE) as usize + i * MR_EXTEND_GRANULARITY; - h.update(&self.fw[chunk_offset..chunk_offset + MR_EXTEND_GRANULARITY]); + let chunk = self + .fw + .get(chunk_offset..chunk_offset + MR_EXTEND_GRANULARITY) + .context("TDVF section data is outside the firmware")?; + h.update(chunk); } } + Ok(()) }; for s in &self.sections { @@ -285,16 +376,16 @@ impl<'a> Tdvf<'a> { match variant { PageAddOrder::TwoPass => { for page in 0..num_pages { - mem_page_add(&mut h, s, page); + mem_page_add(&mut h, s, page)?; } for page in 0..num_pages { - mr_extend(&mut h, s, page); + mr_extend(&mut h, s, page)?; } } PageAddOrder::SinglePass => { for page in 0..num_pages { - mem_page_add(&mut h, s, page); - mr_extend(&mut h, s, page); + mem_page_add(&mut h, s, page)?; + mr_extend(&mut h, s, page)?; } } } @@ -442,7 +533,11 @@ impl<'a> Tdvf<'a> { let mut td_hob_base_addr = 0x809000u64; for s in &self.sections { if let TDVF_SECTION_TD_HOB | TDVF_SECTION_TEMP_MEM = s.sec_type { - memory_acceptor.accept(s.memory_address, s.memory_address + s.memory_data_size); + let end = s + .memory_address + .checked_add(s.memory_data_size) + .context("TDVF section guest address wraps")?; + memory_acceptor.accept(s.memory_address, end); } if s.sec_type == TDVF_SECTION_TD_HOB { td_hob_base_addr = s.memory_address; @@ -500,7 +595,9 @@ impl<'a> Tdvf<'a> { add_memory_resource_hob(0x07, last_start, last_end - last_start); } - let end_of_hob_list = td_hob_base_addr + td_hob.len() as u64 + 8; + let end_of_hob_list = td_hob_base_addr + .checked_add(td_hob.len() as u64 + 8) + .context("TD HOB end-of-list address overflows")?; td_hob[48..56].copy_from_slice(&end_of_hob_list.to_le_bytes()); Ok(measure_sha384(&td_hob)) @@ -593,4 +690,180 @@ mod tests { ); Ok(()) } + + const TDX_METADATA_OFFSET_GUID: &str = "e47a6535-984a-4798-865e-4685a7bf8ec2"; + const TABLE_FOOTER_GUID: &str = "96b582de-1fb2-45f7-baea-a366c55a082d"; + const BYTES_AFTER_TABLE_FOOTER: usize = 32; + const GUID_TABLE_HEADER_SIZE: usize = 18; + + /// Runs `f` on a worker thread so that a parser which never terminates + /// fails on a deadline instead of hanging the whole test run. The blobs + /// here are attacker-shaped, and a missing termination guard shows up as a + /// hang rather than as a wrong answer. + fn within_deadline(what: &str, f: impl FnOnce() -> T + Send + 'static) -> T { + use std::sync::mpsc::{channel, RecvTimeoutError}; + + let (tx, rx) = channel(); + std::thread::spawn(move || { + let _ = tx.send(f()); + }); + match rx.recv_timeout(std::time::Duration::from_secs(5)) { + Ok(value) => value, + Err(RecvTimeoutError::Timeout) => panic!("{what} did not finish within 5s"), + Err(RecvTimeoutError::Disconnected) => panic!("{what} panicked"), + } + } + + fn parse_within_deadline(fw: Vec) -> Result<(), String> { + within_deadline("Tdvf::parse", move || { + Tdvf::parse(&fw) + .map(|_| ()) + .map_err(|err| format!("{err:#}")) + }) + } + + /// Closes `payload` the way a real image does: the GUIDed table, a footer + /// entry whose size covers the table including itself, and the 32-byte + /// reset vector. + fn fw_with_guid_table(payload: &[u8], table: &[u8]) -> Vec { + let mut fw = payload.to_vec(); + fw.extend_from_slice(table); + fw.extend_from_slice(&((table.len() + GUID_TABLE_HEADER_SIZE) as u16).to_le_bytes()); + fw.extend_from_slice(&encode_guid(TABLE_FOOTER_GUID).unwrap()); + fw.extend_from_slice(&[0u8; BYTES_AFTER_TABLE_FOOTER]); + fw + } + + /// A real entry size covers its own 18-byte header, so zero is impossible + /// -- and it never advances the backwards walk over the table. + #[test] + fn parse_rejects_a_zero_length_guid_table_entry() { + let mut table = 0u16.to_le_bytes().to_vec(); + table.extend_from_slice(&[0u8; 16]); // any GUID but the metadata one + let fw = fw_with_guid_table(&[0u8; 64], &table); + + let err = parse_within_deadline(fw).expect_err("a zero-length entry must be rejected"); + assert!(err.contains("entry length"), "unexpected error: {err}"); + } + + /// A firmware whose table simply has no metadata entry must report that, + /// not walk off the front of the table. + #[test] + fn parse_reports_a_table_without_the_metadata_entry() { + let mut table = 22u16.to_le_bytes().to_vec(); + table.extend_from_slice(&[0u8; 16]); + let mut entry = vec![0u8; 4]; + entry.extend_from_slice(&table); + let fw = fw_with_guid_table(&[0u8; 64], &entry); + + let err = + parse_within_deadline(fw).expect_err("a table without the entry must be rejected"); + assert!( + err.contains("Missing TDVF metadata"), + "unexpected error: {err}" + ); + } + + fn mrtd_within_deadline(fw: Vec) -> Result<(), String> { + within_deadline("Tdvf::parse + mrtd", move || { + let tdvf = Tdvf::parse(&fw).map_err(|err| format!("{err:#}"))?; + tdvf.mrtd_two_pass() + .map(|_| ()) + .map_err(|err| format!("{err:#}")) + }) + } + + fn section_record( + data_offset: u32, + raw_data_size: u32, + memory_address: u64, + memory_data_size: u64, + sec_type: u32, + attributes: u32, + ) -> Vec { + let mut out = Vec::with_capacity(TDVF_SECTION_SIZE); + out.extend_from_slice(&data_offset.to_le_bytes()); + out.extend_from_slice(&raw_data_size.to_le_bytes()); + out.extend_from_slice(&memory_address.to_le_bytes()); + out.extend_from_slice(&memory_data_size.to_le_bytes()); + out.extend_from_slice(&sec_type.to_le_bytes()); + out.extend_from_slice(&attributes.to_le_bytes()); + out + } + + /// Builds a blob the parser accepts: `payload_len` bytes of section data, a + /// TDVF descriptor declaring `declared_sections`, the raw `sections` + /// records, and a GUIDed table pointing back at the descriptor. + fn tdvf_fw(payload_len: usize, sections: &[u8], declared_sections: u32) -> Vec { + let mut payload = vec![0u8; payload_len]; + let meta_offset = payload.len(); + payload.extend_from_slice(b"TDVF"); + payload.extend_from_slice(&(16 + sections.len() as u32).to_le_bytes()); + payload.extend_from_slice(&1u32.to_le_bytes()); // version + payload.extend_from_slice(&declared_sections.to_le_bytes()); + payload.extend_from_slice(sections); + + let offset_field = payload.len(); + let mut table = vec![0u8; 4]; // the descriptor's offset from the end + table.extend_from_slice(&((4 + GUID_TABLE_HEADER_SIZE) as u16).to_le_bytes()); + table.extend_from_slice(&encode_guid(TDX_METADATA_OFFSET_GUID).unwrap()); + + let mut fw = fw_with_guid_table(&payload, &table); + let offset_from_end = (fw.len() - meta_offset) as u32; + fw[offset_field..offset_field + 4].copy_from_slice(&offset_from_end.to_le_bytes()); + fw + } + + /// `num_sections` is a raw `u32` from the file, so it can claim far more + /// sections than the blob holds. The records here are all-zero ones, which + /// pass every per-section check, so nothing but the table limit stops the + /// walk early. + #[test] + fn parse_rejects_more_sections_than_the_blob_holds() { + let fw = tdvf_fw(0x40, &vec![0u8; TDVF_SECTION_SIZE * 200], u32::MAX); + + let err = + parse_within_deadline(fw).expect_err("an impossible section count must be rejected"); + assert!(err.contains("sections"), "unexpected error: {err}"); + } + + /// MR.EXTEND hashes `memory_data_size` bytes from `data_offset`, which only + /// `memory_data_size >= raw_data_size` used to constrain. + #[test] + fn parse_rejects_a_section_whose_data_runs_past_the_blob() { + let section = section_record(0, 0x1000, 0x1000, 0x100000, 0, ATTRIBUTE_MR_EXTEND); + let fw = tdvf_fw(0x1000, §ion, 1); + + let err = mrtd_within_deadline(fw).expect_err("an out-of-range section must be rejected"); + assert!(err.contains("firmware"), "unexpected error: {err}"); + } + + /// `memory_data_size` is a page-aligned `u64` from the file, and it alone + /// decides how many pages get hashed. + #[test] + fn parse_rejects_a_section_larger_than_any_real_firmware_measures() { + let section = section_record(0, 0, 0x800000, 0x1000_0000_0000, TDVF_SECTION_TEMP_MEM, 0); + let fw = tdvf_fw(0x40, §ion, 1); + + let err = mrtd_within_deadline(fw).expect_err("an oversized section must be rejected"); + assert!(err.contains("pages"), "unexpected error: {err}"); + } + + /// The measured guest address is `memory_address + page * PAGE_SIZE`, and + /// both halves come from the file. + #[test] + fn parse_rejects_a_section_whose_guest_address_wraps() { + let section = section_record( + 0, + 0, + 0xffff_ffff_ffff_f000, + 0x2000, + TDVF_SECTION_TEMP_MEM, + 0, + ); + let fw = tdvf_fw(0x40, §ion, 1); + + let err = mrtd_within_deadline(fw).expect_err("a wrapping section must be rejected"); + assert!(err.contains("wraps"), "unexpected error: {err}"); + } } diff --git a/dstack/dstack-mr/src/tdx.rs b/dstack/dstack-mr/src/tdx.rs index 72e29f405..96653de06 100644 --- a/dstack/dstack-mr/src/tdx.rs +++ b/dstack/dstack-mr/src/tdx.rs @@ -15,7 +15,9 @@ use crate::kernel::{ tdx_kernel_hash_uses_precomputed_high_mem, TDX_KERNEL_HASH_COMPAT_2G_MEMORY, TDX_KERNEL_HASH_STABLE_MIN_MEMORY, }; -use crate::tdvf::{rtmr0_log_from_td_hob_hash_with_acpi_hashes, AcpiTableHashes, Tdvf}; +use crate::tdvf::{ + rtmr0_log_from_td_hob_hash_with_acpi_hashes, AcpiTableHashes, Tdvf, MAX_TDVF_SECTIONS, +}; use crate::util::{measure_log, measure_sha384}; use anyhow::{bail, Context, Result}; use dstack_types::{ @@ -146,6 +148,9 @@ fn read_varuint(input: &mut &[u8]) -> Result { .split_first() .context("truncated TD HOB witness varuint")?; *input = rest; + if shift == 63 && byte & 0x7e != 0 { + bail!("TD HOB witness varuint is too large"); + } value |= ((byte & 0x7f) as u64) << shift; if byte & 0x80 == 0 { return Ok(value); @@ -163,26 +168,44 @@ fn read_varuint(input: &mut &[u8]) -> Result { /// ranges follow that split. const Q35_HIGH_MEMORY_SPLIT: u64 = 0xB000_0000; +/// Largest number of accepted ranges a TD HOB witness may declare. The witness +/// carries one range per TD_HOB/TEMP_MEM section -- four in the real metadata +/// -- so the TDVF section-table limit is an upper bound no firmware the parser +/// accepts can reach. On the TDX-lite path the witness bytes come from +/// `vm_config.tdx_measurement`, which the requester authors, and a range costs +/// about two bytes there while `MemoryAcceptor::accept` rebuilds and re-sorts +/// the whole range vector per call: a 1 MiB body is a quarter-million ranges +/// and O(n^2 log n) work. +const MAX_TD_HOB_WITNESS_RANGES: u64 = MAX_TDVF_SECTIONS as u64; + fn measure_td_hob_from_witness_data(data: &[u8], memory_size: u64) -> Result> { let mut input = data; let base_page = read_varuint(&mut input)?; let td_hob_page_delta = read_varuint(&mut input)?; let range_count = read_varuint(&mut input)?; - let td_hob_base_addr = (base_page + td_hob_page_delta) - .checked_mul(0x1000) + if range_count > MAX_TD_HOB_WITNESS_RANGES { + bail!("TD HOB witness declares {range_count} ranges, more than the {MAX_TD_HOB_WITNESS_RANGES} supported"); + } + let td_hob_base_addr = base_page + .checked_add(td_hob_page_delta) + .and_then(|page| page.checked_mul(0x1000)) .context("TD HOB base address overflow")?; let mut memory_acceptor = MemoryAcceptor::new(0, memory_size); for _ in 0..range_count { let start_page_delta = read_varuint(&mut input)?; let page_count = read_varuint(&mut input)?; - let start = (base_page + start_page_delta) - .checked_mul(0x1000) + let start = base_page + .checked_add(start_page_delta) + .and_then(|page| page.checked_mul(0x1000)) .context("TD HOB range start overflow")?; let len = page_count .checked_mul(0x1000) .context("TD HOB range length overflow")?; - memory_acceptor.accept(start, start + len); + let end = start + .checked_add(len) + .context("TD HOB range end overflow")?; + memory_acceptor.accept(start, end); } if !input.is_empty() { bail!("TD HOB witness has trailing bytes"); @@ -240,7 +263,9 @@ fn measure_td_hob_from_witness_data(data: &[u8], memory_size: u64) -> Result) { + loop { + let mut byte = (value & 0x7f) as u8; + value >>= 7; + if value != 0 { + byte |= 0x80; + } + out.push(byte); + if value == 0 { + return; + } + } + } + + #[test] + fn read_varuint_rejects_values_larger_than_u64() { + let mut max = Vec::new(); + put_varuint(u64::MAX, &mut max); + assert_eq!(read_varuint(&mut max.as_slice()).unwrap(), u64::MAX); + + let mut bytes = [0xff; 10]; + bytes[9] = 0x02; + let mut encoded = bytes.as_slice(); + + let err = read_varuint(&mut encoded).expect_err("an overflowing varuint must be rejected"); + assert!( + err.to_string().contains("too large"), + "unexpected error: {err}" + ); + } + + fn witness_within_deadline(witness: Vec, memory_size: u64) -> Result<(), String> { + use std::sync::mpsc::{channel, RecvTimeoutError}; + + let (tx, rx) = channel(); + std::thread::spawn(move || { + let _ = tx.send( + measure_td_hob_from_witness_data(&witness, memory_size) + .map(|_| ()) + .map_err(|err| format!("{err:#}")), + ); + }); + match rx.recv_timeout(std::time::Duration::from_secs(5)) { + Ok(result) => result, + Err(RecvTimeoutError::Timeout) => { + panic!("witness measurement did not finish within 5s") + } + Err(RecvTimeoutError::Disconnected) => panic!("witness measurement panicked"), + } + } + + /// The witness is decoded from `vm_config.tdx_measurement`, which the + /// requester supplies, and every range costs about two bytes. A body an + /// attacker can post in one request must not turn into quadratic work. + #[test] + fn measure_td_hob_from_witness_data_rejects_more_ranges_than_a_firmware_has_sections() { + const RANGES: u64 = 50_000; + + let mut witness = Vec::new(); + put_varuint(0x800, &mut witness); // base_page + put_varuint(0, &mut witness); // td_hob_page_delta + put_varuint(RANGES, &mut witness); // range_count + for i in 0..RANGES { + put_varuint(i * 2, &mut witness); // start_page_delta + put_varuint(1, &mut witness); // page_count + } + + let err = witness_within_deadline(witness, 2 * 1024 * 1024 * 1024) + .expect_err("an oversized range count must be rejected"); + assert!(err.contains("ranges"), "unexpected error: {err}"); + } + + /// Every address in the witness is a varint the requester chose, so the + /// page-to-address arithmetic has to survive values no generator emits. + #[test] + fn measure_td_hob_from_witness_data_rejects_ranges_whose_addresses_wrap() { + let with_range = |base_page: u64, start_page_delta: u64, page_count: u64| { + let mut witness = Vec::new(); + put_varuint(base_page, &mut witness); + put_varuint(0, &mut witness); // td_hob_page_delta + put_varuint(1, &mut witness); // range_count + put_varuint(start_page_delta, &mut witness); + put_varuint(page_count, &mut witness); + witness + }; + + // `base_page + start_page_delta` wraps. + let err = witness_within_deadline(with_range(0x1000, u64::MAX, 1), 2 * 1024 * 1024 * 1024) + .expect_err("a wrapping range start must be rejected"); + assert!(err.contains("overflow"), "unexpected error: {err}"); + + // The start is representable, but `start + len` wraps. + let err = witness_within_deadline( + with_range(0x000f_ffff_ffff_ffff, 0, 2), + 2 * 1024 * 1024 * 1024, + ) + .expect_err("a wrapping range end must be rejected"); + assert!(err.contains("overflow"), "unexpected error: {err}"); + } } diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 944bf12e1..5317c9bcb 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -1524,12 +1524,28 @@ fn cbor_to_vec(value: &T, context: &str) -> Vec { out } +/// Decode one CBOR document, and only one. +/// +/// Every caller is decoding a fixed-size measurement document whose bytes are +/// also hashed into `os_image_hash`, so a decoder that stops at the end of the +/// first item and ignores the rest lets one byte string decode to a value it +/// does not hash as. Nothing exploits that today because the verification path +/// hashes the raw bytes (`verify_measurement_material`), but the ambiguity is +/// cheap to remove here and expensive to notice later. fn cbor_from_slice( bytes: &[u8], context: &str, ) -> Result { - ciborium::de::from_reader(Cursor::new(bytes)) - .map_err(|e| format!("{context}: failed to decode CBOR: {e}")) + let mut cursor = Cursor::new(bytes); + let value = ciborium::de::from_reader(&mut cursor) + .map_err(|e| format!("{context}: failed to decode CBOR: {e}"))?; + let trailing = bytes.len() as u64 - cursor.position(); + if trailing != 0 { + return Err(format!( + "{context}: {trailing} trailing byte(s) after the CBOR document" + )); + } + Ok(value) } fn sha256(bytes: &[u8]) -> [u8; 32] { @@ -1540,6 +1556,7 @@ fn sha256(bytes: &[u8]) -> [u8; 32] { pub const TDX_MEASUREMENT_FILENAME: &str = "measurement.tdx.cbor"; pub const SNP_MEASUREMENT_FILENAME: &str = "measurement.snp.cbor"; pub const GCP_MEASUREMENT_FILENAME: &str = "measurement.gcp.cbor"; +pub const AWS_MEASUREMENT_FILENAME: &str = "measurement.aws.cbor"; pub fn image_hash_from_sha256sum(checksum_file: &[u8]) -> [u8; 32] { sha256(checksum_file) @@ -1747,7 +1764,32 @@ pub struct AwsOsImageMeasurement { pub boot_pcr_digest: Vec, } +/// On-wire mirror of [`AwsOsImageMeasurement`], matching +/// `CborGcpOsImageMeasurement`: the document names its schema version so a +/// future shape fails closed on an old verifier instead of decoding as this +/// one. Like its three siblings it does not `deny_unknown_fields`: the version +/// gate below is the fail-closed mechanism, and rejecting on the unknown field +/// first would replace its diagnostic with `unknown field ...`. The bytes are +/// bound to `os_image_hash` by the `measurement.aws.cbor` entry in +/// `sha256sum.txt`, so an ignored field cannot change what was measured. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct CborAwsOsImageMeasurement { + version: u32, + #[serde(with = "hex_bytes")] + boot_pcr_digest: Vec, +} + +impl From<&AwsOsImageMeasurement> for CborAwsOsImageMeasurement { + fn from(measurement: &AwsOsImageMeasurement) -> Self { + Self { + version: AwsOsImageMeasurement::VERSION, + boot_pcr_digest: measurement.boot_pcr_digest.clone(), + } + } +} + impl AwsOsImageMeasurement { + pub const VERSION: u32 = 1; pub const BOOT_PCR_DIGEST_LEN: usize = 32; pub const PCR_SHA384_LEN: usize = 48; @@ -1782,13 +1824,31 @@ impl AwsOsImageMeasurement { } pub fn to_cbor_vec(&self) -> Vec { - cbor_to_vec(self, "AwsOsImageMeasurement") + cbor_to_vec( + &CborAwsOsImageMeasurement::from(self), + "AwsOsImageMeasurement", + ) } pub fn from_cbor_slice(bytes: &[u8]) -> Result { - let measurement: Self = cbor_from_slice(bytes, "AwsOsImageMeasurement")?; + let measurement: CborAwsOsImageMeasurement = + cbor_from_slice(bytes, "AwsOsImageMeasurement")?; + if measurement.version != Self::VERSION { + return Err(format!( + "AwsOsImageMeasurement: unsupported version {}, expected {}", + measurement.version, + Self::VERSION + )); + } Self::new(measurement.boot_pcr_digest) } + + pub fn cbor_json_value_from_slice(bytes: &[u8]) -> Result { + let measurement: CborAwsOsImageMeasurement = + cbor_from_slice(bytes, "AwsOsImageMeasurement")?; + serde_json::to_value(measurement) + .map_err(|e| format!("AwsOsImageMeasurement: failed to convert to JSON: {e}")) + } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -1819,7 +1879,7 @@ impl AwsOsImageMeasurementDocument { os_image_hash, &self.checksum_file, &self.measurement, - "measurement.aws.cbor", + AWS_MEASUREMENT_FILENAME, ) } } @@ -2876,3 +2936,88 @@ mod tdx_measurement_cbor_tests { assert!(err.contains("unsupported version 2"), "unexpected: {err}"); } } + +#[cfg(test)] +mod cbor_canonicalization_tests { + use super::*; + + fn sev_measurement() -> SevOsImageMeasurement { + SevOsImageMeasurement { + base_cmdline: "console=ttyS0 dstack.rootfs_hash=11".to_string(), + ovmf_hash: vec![0x44; 48], + kernel_hash: vec![0x55; 32], + initrd_hash: vec![0x66; 32], + sev_hashes_table_gpa: 0x80_1000, + sev_es_reset_eip: 0xffff_fff0, + ovmf_sections: vec![OvmfSection { + gpa: 0x100000, + size: 0x1000, + section_type: 1, + }], + } + } + + /// Every `*_MEASUREMENT_FILENAME` document is bound to `os_image_hash` by a + /// hash over its raw bytes, so today the trailing bytes ride along in that + /// hash and no live check is bypassed. That property is an accident of the + /// call sites, not of the decoder: the moment anything compares a + /// re-encoded `measurement_hash()` against a `sha256sum.txt` entry, a + /// document with trailing bytes decodes to one value and hashes as another. + /// Decoding is where the ambiguity belongs. + #[test] + fn cbor_decoders_reject_trailing_bytes() { + let mut cbor = sev_measurement().to_cbor_vec(); + let clean = SevOsImageMeasurement::from_cbor_slice(&cbor).expect("clean document decodes"); + cbor.push(0x00); + + let err = SevOsImageMeasurement::from_cbor_slice(&cbor) + .expect_err("a document with trailing bytes must not decode"); + assert!(err.contains("trailing"), "unexpected error: {err}"); + + // The re-encoded hash of what it decodes to is not the hash of the + // bytes it was decoded from -- the ambiguity the check removes. + assert_ne!(clean.measurement_hash().to_vec(), sha256(&cbor).to_vec()); + + // Same decoder, so every other measurement document is covered too. + let mut gcp = GcpOsImageMeasurement::new(vec![0x77; 32]) + .expect("gcp measurement") + .to_cbor_vec(); + gcp.extend_from_slice(b"junk"); + let err = GcpOsImageMeasurement::from_cbor_slice(&gcp) + .expect_err("a GCP document with trailing bytes must not decode"); + assert!(err.contains("trailing"), "unexpected error: {err}"); + } + + /// Each of `measurement.{tdx,snp,gcp}.cbor` encodes a mirror struct whose + /// first field is `version`, checked on decode, so a future schema change + /// fails closed on old verifiers. `measurement.aws.cbor` encoded its public + /// struct directly and named no version at all, which would have left a v2 + /// document decoding as a v1 one. + #[test] + fn the_aws_measurement_document_names_and_checks_its_version() { + let cbor = AwsOsImageMeasurement::new(vec![0x77; 32]) + .expect("aws measurement") + .to_cbor_vec(); + + let value: ciborium::value::Value = + ciborium::de::from_reader(Cursor::new(&cbor[..])).expect("document decodes as CBOR"); + let entries = value.as_map().expect("document is a CBOR map"); + let version = entries + .iter() + .find(|(key, _)| key.as_text() == Some("version")) + .map(|(_, value)| value.clone()) + .expect("measurement.aws.cbor names a version, like its three siblings"); + assert_eq!(version.as_integer(), Some(1u32.into())); + + let forged = cbor_to_vec( + &CborAwsOsImageMeasurement { + version: 2, + boot_pcr_digest: vec![0x77; 32], + }, + "AwsOsImageMeasurement", + ); + let err = AwsOsImageMeasurement::from_cbor_slice(&forged) + .expect_err("a v2 document must not decode as v1"); + assert!(err.contains("unsupported version 2"), "unexpected: {err}"); + } +} diff --git a/dstack/dstack-types/src/mr_config.rs b/dstack/dstack-types/src/mr_config.rs index ce38f4e7c..d11c3a5aa 100644 --- a/dstack/dstack-types/src/mr_config.rs +++ b/dstack/dstack-types/src/mr_config.rs @@ -105,7 +105,9 @@ impl From for MrConfigDocumentError { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct MrConfigV3 { - #[serde(default = "mr_config_v3_version")] + /// Document schema version, always 3. No serde default: `validate_mr_config` + /// rejects anything but 3, and a default would have made an absent version + /// mean 3 rather than fail that gate. pub version: u8, /// Optional application identity pin. #[serde(default, with = "hex_bytes")] @@ -251,7 +253,7 @@ mod tests { #[test] fn mr_config_v3_defaults_missing_app_id_to_empty() -> Result<(), Box> { let config = MrConfigV3::from_document( - r#"{"compose_hash":"2222222222222222222222222222222222222222222222222222222222222222","key_provider":"none"}"#, + r#"{"version":3,"compose_hash":"2222222222222222222222222222222222222222222222222222222222222222","key_provider":"none"}"#, )?; assert!(config.app_id.is_none()); @@ -316,4 +318,21 @@ mod tests { ); Ok(()) } + + /// `validate_mr_config` gates on `version != 3`, which only means anything + /// if an absent `version` is absent. A serde default made every document + /// that never mentions a version pass that gate as a v3 document. + #[test] + fn a_document_without_a_version_is_not_a_v3_document() { + let err = MrConfigV3::from_document(concat!( + "{\"compose_hash\":", + "\"2222222222222222222222222222222222222222222222222222222222222222\",", + "\"key_provider\":\"none\"}" + )) + .expect_err("a document without a version must not parse as v3"); + assert!( + err.to_string().contains("version"), + "unexpected error: {err}" + ); + } } diff --git a/dstack/dstack-util/src/docker_compose.rs b/dstack/dstack-util/src/docker_compose.rs index 7da2a5b78..44bcd99be 100644 --- a/dstack/dstack-util/src/docker_compose.rs +++ b/dstack/dstack-util/src/docker_compose.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use bollard::container::{ListContainersOptions, RemoveContainerOptions}; use bollard::Docker; use fs_err as fs; @@ -19,6 +19,12 @@ pub struct ComposeInfo { } /// Parse a docker-compose file and extract project name and service names +/// +/// The caller deletes containers whose service is absent from `service_names`, +/// so a service this parser cannot see is a live container it destroys. That +/// makes an incomplete answer worse than no answer, and `include:` merges in +/// services from files that are not read here at all -- Compose starts them and +/// labels them with this same project. Refuse rather than guess. pub fn parse_docker_compose_file(compose_file: impl AsRef) -> Result { let compose_content = fs::read_to_string(compose_file.as_ref()).context("failed to read docker-compose file")?; @@ -26,6 +32,10 @@ pub fn parse_docker_compose_file(compose_file: impl AsRef) -> Result // SPDX-License-Identifier: Apache-2.0 +use std::ffi::OsString; use std::path::{Path, PathBuf}; use std::process::Command; @@ -60,6 +61,35 @@ fn find_disk_by_label(label: &str) -> Option { None } +/// Mount options for the host share, on both the disk and the 9p path. +/// +/// The share is a filesystem the host composes, and in dstack's threat model +/// the host is the adversary. Nothing on it is ever executed and nothing on it +/// is a device node: the guest copies five regular files off it as root and +/// unmounts it. So denying set-user-ID bits, device nodes and execution costs +/// nothing and removes the corresponding paths entirely. +const HOST_SHARED_MOUNT_OPTIONS: &str = "ro,nosuid,nodev,noexec"; + +fn disk_mount_args(device: &Path, mount_point: &Path) -> Vec { + vec![ + "-o".into(), + HOST_SHARED_MOUNT_OPTIONS.into(), + device.into(), + mount_point.into(), + ] +} + +fn p9_mount_args(mount_point: &Path) -> Vec { + vec![ + "-t".into(), + "9p".into(), + "-o".into(), + format!("trans=virtio,version=9p2000.L,{HOST_SHARED_MOUNT_OPTIONS}").into(), + "host-shared".into(), + mount_point.into(), + ] +} + pub fn mount_host_shared(mount_point: &Path) -> Result<()> { fs::create_dir_all(mount_point) .with_context(|| format!("failed to create {}", mount_point.display()))?; @@ -67,9 +97,7 @@ pub fn mount_host_shared(mount_point: &Path) -> Result<()> { if let Some(device) = find_disk_by_label(HOST_SHARED_DISK_LABEL) { info!(device = %device.display(), "found host-shared disk"); let status = Command::new("mount") - .args(["-o", "ro"]) - .arg(&device) - .arg(mount_point) + .args(disk_mount_args(&device, mount_point)) .status() .with_context(|| format!("failed to run mount for {}", device.display()))?; if status.success() { @@ -86,14 +114,7 @@ pub fn mount_host_shared(mount_point: &Path) -> Result<()> { } let status = Command::new("mount") - .args([ - "-t", - "9p", - "-o", - "trans=virtio,version=9p2000.L,ro", - "host-shared", - ]) - .arg(mount_point) + .args(p9_mount_args(mount_point)) .status() .context("failed to run 9p mount")?; anyhow::ensure!( @@ -129,6 +150,35 @@ pub fn cmd_host_shared(args: HostSharedArgs) -> Result<()> { mod tests { use super::*; + /// The host composes this filesystem. Neither path may let it ship a + /// set-user-ID binary, a device node or anything executable into the guest. + #[test] + fn both_mount_paths_refuse_suid_devices_and_execution() { + let disk = disk_mount_args(Path::new("/dev/vdb"), Path::new("/dstack/.host-shared")); + assert_eq!( + disk, + [ + "-o", + "ro,nosuid,nodev,noexec", + "/dev/vdb", + "/dstack/.host-shared" + ] + ); + + let p9 = p9_mount_args(Path::new("/dstack/.host-shared")); + assert_eq!( + p9, + [ + "-t", + "9p", + "-o", + "trans=virtio,version=9p2000.L,ro,nosuid,nodev,noexec", + "host-shared", + "/dstack/.host-shared", + ] + ); + } + #[test] fn parses_mount_command() { let args = HostSharedArgs::try_parse_from([ diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index 2a456ab12..4b37b74cf 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -282,20 +282,25 @@ impl HostShared { } bail!("Source file {src} does not exist"); } - let src_size = src_path.metadata()?.len(); - if src_size > max_size { - bail!("Source file {src} is too large, max size is {max_size} bytes"); - } use fs::os::unix::fs::OpenOptionsExt; - let mut src_io = fs::OpenOptions::new() + use std::io::Read; + // O_NONBLOCK: opening a FIFO on the host share must not block the boot. + let src_io = fs::OpenOptions::new() .read(true) - .custom_flags(libc::O_NOFOLLOW) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) .open(src_path)?; + if !src_io.metadata()?.is_file() { + bail!("Source file {src} is not a regular file"); + } let mut dst_io = fs::OpenOptions::new() .write(true) .create(true) + .truncate(true) .open(dst_path)?; - std::io::copy(&mut src_io, &mut dst_io)?; + let copied = std::io::copy(&mut src_io.take(max_size + 1), &mut dst_io)?; + if copied > max_size { + bail!("Source file {src} is too large, max size is {max_size} bytes"); + } Ok(()) }; info!("Mounting host-shared"); diff --git a/dstack/dstack-util/src/system_setup/config_id_verifier.rs b/dstack/dstack-util/src/system_setup/config_id_verifier.rs index 3e71cd76c..fdb00c034 100644 --- a/dstack/dstack-util/src/system_setup/config_id_verifier.rs +++ b/dstack/dstack-util/src/system_setup/config_id_verifier.rs @@ -48,7 +48,7 @@ fn read_snp_host_data() -> Result<[u8; 32]> { let AttestationQuote::DstackAmdSevSnp(quote) = attestation.quote else { bail!("attestation mode is not AMD SEV-SNP"); }; - let parsed = dstack_attest::amd_sev_snp::parse_amd_snp_report("e.report) + let parsed = dstack_attest::amd_sev_snp::parse_unverified_amd_snp_report("e.report) .context("Failed to parse SNP report")?; Ok(parsed.host_data) } diff --git a/dstack/gateway/dstack-app/builder/README.md b/dstack/gateway/dstack-app/builder/README.md index 18e074452..b8aedad53 100644 --- a/dstack/gateway/dstack-app/builder/README.md +++ b/dstack/gateway/dstack-app/builder/README.md @@ -59,17 +59,17 @@ GIT_REV=HEAD \ IMAGE_VERSION=0.6.0 \ IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack \ OCI_TAR=/tmp/gateway.oci.tar \ - ./build-image.sh dstacktee/dstack-gateway:0.6.0 + ./build-image.sh ghcr.io/dstack-tee/dstack-gateway:0.6.0 python3 -c 'import json,tarfile;t=tarfile.open("/tmp/gateway.oci.tar");print(json.load(t.extractfile("index.json"))["manifests"][0]["digest"])' ``` `IMAGE_VERSION` is part of the image metadata, so it must match the release for the digests to match. The printed digest is what the registry reports for -`dstacktee/dstack-gateway:0.6.0`; compare it with: +`ghcr.io/dstack-tee/dstack-gateway:0.6.0`; compare it with: ```bash -docker buildx imagetools inspect dstacktee/dstack-gateway:0.6.0 --format '{{.Manifest.Digest}}' +docker buildx imagetools inspect ghcr.io/dstack-tee/dstack-gateway:0.6.0 --format '{{.Manifest.Digest}}' ``` ## Image metadata diff --git a/dstack/gateway/src/pp.rs b/dstack/gateway/src/pp.rs index 893e3f844..ab9de2829 100644 --- a/dstack/gateway/src/pp.rs +++ b/dstack/gateway/src/pp.rs @@ -95,6 +95,23 @@ fn create_inbound_pp_header(inbound: &TcpStream) -> ProxyHeader { } } +/// Read a PROXY protocol header and parse it. +/// +/// The amount of data handed to `proxy_protocol::parse` is load-bearing. +/// `version2::parse` checks `buf.remaining()` against the *fixed* size of the +/// address family and then does `buf.advance(length - that)` using the +/// attacker-declared `length` -- an unchecked advance that panics inside +/// `bytes`, which with `panic = "abort"` would be the whole gateway. Handing it +/// a 228-byte AF_UNIX header declaring `length = 65535` produces +/// `advance out of bounds: the len is 0 but advancing by 65319`. +/// +/// Both v2 paths call `read_exact` for the complete declared body before +/// parsing. The fixed-buffer path therefore has at least `full_length` bytes +/// available, and the oversize path allocates exactly `full_length` bytes. +/// Keep that full-body read if this is changed to parse incrementally: calling +/// `parse` with fewer than the declared number of bytes can reach the unchecked +/// advance. `representative_v2_header_lengths_do_not_abort` covers the boundary +/// lengths on both sides of the fixed-buffer cutoff. async fn read_proxy_header(mut stream: I) -> Result<(I, ProxyHeader)> where I: AsyncRead + Unpin, @@ -204,6 +221,40 @@ mod tests { } } + /// Representative and boundary v2 lengths against every address family, + /// with the body absent, truncated and complete. The reader must return -- + /// `Ok` or `Err` -- and never reach the unchecked `advance` in + /// `version2::parse`. + #[tokio::test] + async fn representative_v2_header_lengths_do_not_abort() { + for family_proto in [0x00u8, 0x11, 0x21, 0x31, 0x12, 0x99] { + for length in [ + 0u16, 1, 12, 15, 16, 36, 215, 216, 217, 495, 496, 497, 2032, 2047, 65535, + ] { + for supplied in [0usize, 8, 216, usize::from(length)] { + let mut header = V2_PROTOCOL_PREFIX.to_vec(); + header.extend_from_slice(&[0x21, family_proto]); + header.extend_from_slice(&length.to_be_bytes()); + header.extend(std::iter::repeat_n(0x41u8, supplied)); + // Returning at all is the property under test. + let _ = read_proxy_header(&header[..]).await; + } + } + } + } + + /// The v1 reader stops at the first CRLF, and the parser is then handed the + /// whole zero-padded array rather than the bytes actually read -- which is + /// what keeps `version1::parse`'s two unguarded `get_u8` calls at the end of + /// the header in bounds. + #[tokio::test] + async fn a_v1_header_ending_on_a_bare_cr_does_not_abort() { + for tail in ["", " ", " 1", " 1 ", " 1 65535"] { + let header = format!("PROXY TCP4 1.2.3.4 5.6.7.8{tail}\r\n"); + let _ = read_proxy_header(header.as_bytes()).await; + } + } + #[tokio::test] async fn parses_v1_ipv4() { // v1 is ASCII: "PROXY TCP4 \r\n" diff --git a/dstack/gateway/src/proxy/sni.rs b/dstack/gateway/src/proxy/sni.rs index 4901cc45f..986ba9807 100644 --- a/dstack/gateway/src/proxy/sni.rs +++ b/dstack/gateway/src/proxy/sni.rs @@ -10,6 +10,23 @@ pub fn extract_sni(b: &[u8]) -> Option<&[u8]> { extract_sni_inner(b).ok().map(|r| r.1) } +/// Read a `u8`-length-prefixed record. +/// +/// `parcelona::take_record_be_u8` cannot be used here: its guard is +/// `b.len() < b[0]`, but it then splits `&b[1..]` -- one byte shorter -- at +/// `b[0]`, so a record whose declared length equals the number of bytes +/// *including* the length byte slips past the check and panics inside +/// `split_at_revers`. With `panic = "abort"` that is the whole gateway, so this +/// composes the two correctly-guarded primitives instead. `take_record_be_u16` +/// is already written this way upstream and needs no replacement. +/// +/// Reachable from the public TLS port: a 53-byte ClientHello prefix whose +/// session-id length byte is the count of bytes after it plus one is enough. +fn take_record_u8_len(b: &[u8]) -> Result<(&[u8], &[u8]), PErr<'_, u8>> { + let (b, len) = take_len_be_u8(b)?; + take_record(b, len) +} + fn extract_sni_inner(b: &[u8]) -> Result<(usize, &[u8]), PErr<'_, u8>> { const HANDSHAKE_TYPE_CLIENT_HELLO: usize = 1; const EXTENSION_TYPE_SNI: usize = 0; @@ -36,9 +53,9 @@ fn extract_sni_inner(b: &[u8]) -> Result<(usize, &[u8]), PErr<'_, u8>> { let (b, _) = take_record(b, 34)?; // Session ID (u8-length vec), cipher suites (u16-length vec), compression methods (u8-length vec). - let (b, _) = take_record_be_u8(b)?; + let (b, _) = take_record_u8_len(b)?; let (b, _) = take_record_be_u16(b)?; - let (b, _) = take_record_be_u8(b)?; + let (b, _) = take_record_u8_len(b)?; // Extensions length. let (mut b, mut c) = take_len_be_u16(b)?; @@ -82,3 +99,121 @@ fn extract_sni_inner(b: &[u8]) -> Result<(usize, &[u8]), PErr<'_, u8>> { } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A ClientHello carrying one `host_name` SNI entry. + fn client_hello(server_name: &[u8]) -> Vec { + let mut sni_ext = vec![0x00]; // NameType: host_name + sni_ext.extend_from_slice(&(server_name.len() as u16).to_be_bytes()); + sni_ext.extend_from_slice(server_name); + + let mut server_name_list = (sni_ext.len() as u16).to_be_bytes().to_vec(); + server_name_list.extend_from_slice(&sni_ext); + + let mut extensions = vec![0x00, 0x00]; // ExtensionType: server_name + extensions.extend_from_slice(&(server_name_list.len() as u16).to_be_bytes()); + extensions.extend_from_slice(&server_name_list); + + let mut body = vec![0x03, 0x03]; // legacy_version + body.extend_from_slice(&[0x41; 32]); // random + body.push(0x00); // session_id: empty + body.extend_from_slice(&[0x00, 0x02, 0x13, 0x01]); // cipher_suites + body.extend_from_slice(&[0x01, 0x00]); // compression_methods: null + body.extend_from_slice(&(extensions.len() as u16).to_be_bytes()); + body.extend_from_slice(&extensions); + + let mut handshake = vec![0x01]; // client_hello + handshake.extend_from_slice(&(body.len() as u32).to_be_bytes()[1..]); // u24 length + handshake.extend_from_slice(&body); + + let mut record = vec![0x16, 0x03, 0x01]; // handshake record, TLS 1.0 framing + record.extend_from_slice(&(handshake.len() as u16).to_be_bytes()); + record.extend_from_slice(&handshake); + record + } + + #[test] + fn extracts_the_server_name_from_a_client_hello() { + let hello = client_hello(b"app.example.com"); + assert_eq!(extract_sni(&hello), Some(&b"app.example.com"[..])); + } + + #[test] + fn a_truncated_client_hello_yields_no_name_rather_than_a_wrong_one() { + let hello = client_hello(b"app.example.com"); + for len in 0..hello.len() { + assert_eq!(extract_sni(&hello[..len]), None, "prefix of {len} bytes"); + } + } + + /// A `u8`-length-prefixed record may declare a length that covers its own + /// length byte. `parcelona::take_record_be_u8` admits exactly that case and + /// then indexes past the end of the slice; the process builds with + /// `panic = "abort"`, so on the public TLS port one short write ends every + /// connection the gateway is currently relaying, not just this one. + #[test] + fn a_session_id_length_covering_its_own_length_byte_does_not_abort() { + // 5 record header + 1 handshake type + 3 handshake length + 34 + // version/random puts the session-id length byte at offset 43. + const SESSION_ID_LEN_OFFSET: usize = 43; + fn over_declaring_hello(total: usize) -> Vec { + let mut hello = vec![0x16, 0x03, 0x01, 0x00, 0x00]; + hello.push(0x01); + hello.extend_from_slice(&[0x00, 0x00, 0x00]); + hello.extend_from_slice(&[0x00; 34]); + assert_eq!(hello.len(), SESSION_ID_LEN_OFFSET); + // Declares one byte more than actually follows it. + hello.push((total - SESSION_ID_LEN_OFFSET) as u8); + hello.resize(total, 0); + hello + } + + // The minimal reproducer: 53 bytes, session-id length 10, nine to go. + assert_eq!(extract_sni(&over_declaring_hello(53)), None); + for total in SESSION_ID_LEN_OFFSET + 2..=SESSION_ID_LEN_OFFSET + 255 { + // Returning at all is the property; an all-zero tail is a + // well-formed empty extension list for some of these lengths. + let _ = extract_sni(&over_declaring_hello(total)); + } + } + + /// The same shape one level down, where the compression-methods length is + /// the field that over-declares. + #[test] + fn a_compression_methods_length_covering_its_own_length_byte_does_not_abort() { + let mut hello = vec![0x16, 0x03, 0x01, 0x00, 0x00]; + hello.push(0x01); + hello.extend_from_slice(&[0x00, 0x00, 0x00]); + hello.extend_from_slice(&[0x00; 34]); + hello.push(0x00); // session_id: empty + hello.extend_from_slice(&[0x00, 0x00]); // cipher_suites: empty + hello.push(0x04); // compression_methods: declares 4, three bytes follow + hello.extend_from_slice(&[0x00; 3]); + assert_eq!(extract_sni(&hello), None); + } + + /// Every byte of a valid ClientHello, walked through every value, must + /// leave the extractor returning rather than aborting. Deterministic, so a + /// failure is reproducible from the printed offset and value alone. + #[test] + fn no_single_byte_mutation_of_a_client_hello_aborts_the_extractor() { + let hello = client_hello(b"app.example.com"); + for offset in 0..hello.len() { + for value in [0x00u8, 0x01, 0x7f, 0x80, 0xfe, 0xff] { + let mut mutated = hello.clone(); + mutated[offset] = value; + // The result is uninteresting; not aborting is the property. + let _ = extract_sni(&mutated); + } + // Truncating after a mutated length byte is the other half of it. + for value in [0x00u8, 0xff] { + let mut mutated = hello[..offset].to_vec(); + mutated.push(value); + let _ = extract_sni(&mutated); + } + } + } +} diff --git a/dstack/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs index 7d0159cd0..7c1b6e340 100644 --- a/dstack/guest-agent/src/rpc_service.rs +++ b/dstack/guest-agent/src/rpc_service.rs @@ -602,6 +602,16 @@ pub(crate) fn validate_cert_validity( not_before: Option, not_after: Option, ) -> Result<()> { + for (name, value) in [("not_before", not_before), ("not_after", not_after)] { + if let Some(value) = value { + if value > ra_tls::cert::MAX_CERT_VALIDITY_SECS { + anyhow::bail!( + "{name} must not be later than {}", + ra_tls::cert::MAX_CERT_VALIDITY_SECS + ); + } + } + } if let (Some(not_before), Some(not_after)) = (not_before, not_after) { if not_before >= not_after { anyhow::bail!("not_before must be earlier than not_after"); @@ -1789,7 +1799,8 @@ pNs85uhOZE8z2jr8Pg== } #[test] - fn test_tls_certificate_validity_order() { + fn test_tls_certificate_validity() { + let maximum = ra_tls::cert::MAX_CERT_VALIDITY_SECS; assert!(validate_cert_validity(None, None).is_ok()); assert!(validate_cert_validity(Some(10), Some(11)).is_ok()); assert_eq!( @@ -1799,6 +1810,10 @@ pNs85uhOZE8z2jr8Pg== "not_before must be earlier than not_after" ); assert!(validate_cert_validity(Some(12), Some(11)).is_err()); + assert!(validate_cert_validity(None, Some(maximum)).is_ok()); + assert!(validate_cert_validity(Some(0), Some(maximum)).is_ok()); + assert!(validate_cert_validity(None, Some(maximum + 1)).is_err()); + assert!(validate_cert_validity(Some(maximum + 1), None).is_err()); } #[tokio::test] diff --git a/dstack/kms/Cargo.toml b/dstack/kms/Cargo.toml index a35f34e7e..5c616d7a4 100644 --- a/dstack/kms/Cargo.toml +++ b/dstack/kms/Cargo.toml @@ -48,6 +48,7 @@ dstack-verifier = { workspace = true, default-features = false } dstack-mr.workspace = true dstack-attest.workspace = true dstack-api-auth.workspace = true +moka = { workspace = true, features = ["future"] } [dev-dependencies] cc-eventlog.workspace = true diff --git a/dstack/kms/auth-eth-bun/bun.lock b/dstack/kms/auth-eth-bun/bun.lock index 0af400847..a471f6bc8 100644 --- a/dstack/kms/auth-eth-bun/bun.lock +++ b/dstack/kms/auth-eth-bun/bun.lock @@ -5,7 +5,7 @@ "name": "auth-eth-bun", "dependencies": { "@hono/zod-validator": "0.2.2", - "hono": "4.12.34", + "hono": "4.13.5", "viem": "2.31.7", "zod": "3.25.76", }, @@ -16,64 +16,18 @@ "openapi-types": "12.1.3", "oxlint": "0.9.10", "typescript": "5.8.3", - "vitest": "3.2.6", + "vitest": "4.1.11", }, }, }, "packages": { "@adraffy/ens-normalize": ["@adraffy/ens-normalize@1.11.0", "", {}, "sha512-/3DDPKHqqIqxUULp8yP4zODUY1i+2xvVWsv8A79xGWdCAG+8sb0hRh0Rk2QyOJUnnbyPUAZYcpBuRe3nS2OIUg=="], - "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.21.5", "", { "os": "aix", "cpu": "ppc64" }, "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ=="], - - "@esbuild/android-arm": ["@esbuild/android-arm@0.21.5", "", { "os": "android", "cpu": "arm" }, "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg=="], - - "@esbuild/android-arm64": ["@esbuild/android-arm64@0.21.5", "", { "os": "android", "cpu": "arm64" }, "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A=="], - - "@esbuild/android-x64": ["@esbuild/android-x64@0.21.5", "", { "os": "android", "cpu": "x64" }, "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA=="], - - "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.21.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ=="], - - "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.21.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw=="], - - "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.21.5", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g=="], - - "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.21.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ=="], - - "@esbuild/linux-arm": ["@esbuild/linux-arm@0.21.5", "", { "os": "linux", "cpu": "arm" }, "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA=="], - - "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.21.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q=="], - - "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.21.5", "", { "os": "linux", "cpu": "ia32" }, "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg=="], - - "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg=="], - - "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg=="], - - "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.21.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w=="], - - "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA=="], - - "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.21.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A=="], - - "@esbuild/linux-x64": ["@esbuild/linux-x64@0.21.5", "", { "os": "linux", "cpu": "x64" }, "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ=="], - - "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.21.5", "", { "os": "none", "cpu": "x64" }, "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg=="], - - "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.21.5", "", { "os": "openbsd", "cpu": "x64" }, "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow=="], - - "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.21.5", "", { "os": "sunos", "cpu": "x64" }, "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg=="], - - "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.21.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A=="], - - "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.21.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA=="], - - "@esbuild/win32-x64": ["@esbuild/win32-x64@0.21.5", "", { "os": "win32", "cpu": "x64" }, "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw=="], - "@hono/zod-validator": ["@hono/zod-validator@0.2.2", "", { "peerDependencies": { "hono": ">=3.9.0", "zod": "^3.19.1" } }, "sha512-dSDxaPV70Py8wuIU2QNpoVEIOSzSXZ/6/B/h4xA7eOMz7+AarKTSGV8E6QwrdcCbBLkpqfJ4Q2TmBO0eP1tCBQ=="], "@jest/schemas": ["@jest/schemas@29.6.3", "", { "dependencies": { "@sinclair/typebox": "^0.27.8" } }, "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA=="], - "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.4", "", {}, "sha512-VT2+G1VQs/9oz078bLrYbecdZKs912zQlkelYpuf+SXF+QvZDYJlbx/LSx+meSAwdDFnF8FVXW92AVjjkVmgFw=="], + "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.6.0", "", {}, "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw=="], "@noble/ciphers": ["@noble/ciphers@1.3.0", "", {}, "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw=="], @@ -87,6 +41,8 @@ "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], + "@oxc-project/types": ["@oxc-project/types@0.150.0", "", {}, "sha512-rDS5/31E9HfPl/CIzGrn0DOlvBbXFseQ5URJ9sYMfstbKLD/c6Gm9vmRzRGDdAXyOIL4zmO37lc9RIwYqVruZw=="], + "@oxlint/darwin-arm64": ["@oxlint/darwin-arm64@0.9.10", "", { "os": "darwin", "cpu": "arm64" }, "sha512-eOXKZYq5bnCSgDefgM5bzAg+4Fc//Rc4yjgKN8iDWUARweCaChiQXb6TXX8MfEfs6qayEMy6yVj0pqoFz0B1aw=="], "@oxlint/darwin-x64": ["@oxlint/darwin-x64@0.9.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-UeYICDvLUaUOcY+0ugZUEmBMRLP+x8iTgL7TeY6BlpGw2ahbtUOTbyIIRWtr/0O++TnjZ+v8TzhJ9crw6Ij6dg=="], @@ -105,45 +61,37 @@ "@polka/url": ["@polka/url@1.0.0-next.29", "", {}, "sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww=="], - "@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.45.1", "", { "os": "android", "cpu": "arm" }, "sha512-NEySIFvMY0ZQO+utJkgoMiCAjMrGvnbDLHvcmlA33UXJpYBCvlBEbMMtV837uCkS+plG2umfhn0T5mMAxGrlRA=="], - - "@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.45.1", "", { "os": "android", "cpu": "arm64" }, "sha512-ujQ+sMXJkg4LRJaYreaVx7Z/VMgBBd89wGS4qMrdtfUFZ+TSY5Rs9asgjitLwzeIbhwdEhyj29zhst3L1lKsRQ=="], - - "@rollup/rollup-darwin-arm64": ["@rollup/rollup-darwin-arm64@4.45.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-FSncqHvqTm3lC6Y13xncsdOYfxGSLnP+73k815EfNmpewPs+EyM49haPS105Rh4aF5mJKywk9X0ogzLXZzN9lA=="], - - "@rollup/rollup-darwin-x64": ["@rollup/rollup-darwin-x64@4.45.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-2/vVn/husP5XI7Fsf/RlhDaQJ7x9zjvC81anIVbr4b/f0xtSmXQTFcGIQ/B1cXIYM6h2nAhJkdMHTnD7OtQ9Og=="], - - "@rollup/rollup-freebsd-arm64": ["@rollup/rollup-freebsd-arm64@4.45.1", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-4g1kaDxQItZsrkVTdYQ0bxu4ZIQ32cotoQbmsAnW1jAE4XCMbcBPDirX5fyUzdhVCKgPcrwWuucI8yrVRBw2+g=="], + "@rolldown/binding-android-arm-eabi": ["@rolldown/binding-android-arm-eabi@1.2.9", "", { "os": "android", "cpu": "arm" }, "sha512-tNISae1QEf/vkb3xkRcjV5SEdzPE97We5IVaa2Z8jSszQPZ8U60B/YCYpw4QI7VidYsBtKavczXf+DyDs9WGxw=="], - "@rollup/rollup-freebsd-x64": ["@rollup/rollup-freebsd-x64@4.45.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-L/6JsfiL74i3uK1Ti2ZFSNsp5NMiM4/kbbGEcOCps99aZx3g8SJMO1/9Y0n/qKlWZfn6sScf98lEOUe2mBvW9A=="], + "@rolldown/binding-android-arm64": ["@rolldown/binding-android-arm64@1.2.9", "", { "os": "android", "cpu": "arm64" }, "sha512-YC8YsI30o606GTZi0VyzYlsDKFP8W61i/QzayHDkLbNEz/IShqAmTa+hsJRj13xTHA0H+6fk4b2UmGn+Q/cMlg=="], - "@rollup/rollup-linux-arm-gnueabihf": ["@rollup/rollup-linux-arm-gnueabihf@4.45.1", "", { "os": "linux", "cpu": "arm" }, "sha512-RkdOTu2jK7brlu+ZwjMIZfdV2sSYHK2qR08FUWcIoqJC2eywHbXr0L8T/pONFwkGukQqERDheaGTeedG+rra6Q=="], + "@rolldown/binding-darwin-arm64": ["@rolldown/binding-darwin-arm64@1.2.9", "", { "os": "darwin", "cpu": "arm64" }, "sha512-IwhlH3qK5urrY8hZiEgGkHKEFN901p/p2bjxCxJlr4GyNnF7wYpUvK+Y43uaRYuC4hpfjzbR3SJC3arX1jGvmw=="], - "@rollup/rollup-linux-arm-musleabihf": ["@rollup/rollup-linux-arm-musleabihf@4.45.1", "", { "os": "linux", "cpu": "arm" }, "sha512-3kJ8pgfBt6CIIr1o+HQA7OZ9mp/zDk3ctekGl9qn/pRBgrRgfwiffaUmqioUGN9hv0OHv2gxmvdKOkARCtRb8Q=="], + "@rolldown/binding-darwin-x64": ["@rolldown/binding-darwin-x64@1.2.9", "", { "os": "darwin", "cpu": "x64" }, "sha512-XxpJfVzFh+jilRxIXUqcfYAYcunIc/XEzIizsOL1fcJee5Sf7H3mH8WlLmfHfluz5amqR88QQo9izKtmMlavAw=="], - "@rollup/rollup-linux-arm64-gnu": ["@rollup/rollup-linux-arm64-gnu@4.45.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-k3dOKCfIVixWjG7OXTCOmDfJj3vbdhN0QYEqB+OuGArOChek22hn7Uy5A/gTDNAcCy5v2YcXRJ/Qcnm4/ma1xw=="], + "@rolldown/binding-freebsd-x64": ["@rolldown/binding-freebsd-x64@1.2.9", "", { "os": "freebsd", "cpu": "x64" }, "sha512-kSfvhmgeWyfkbT3p/1s5vSgboogoah2zkm9fX2zjg2hHxSV7T4KhMWRUUaRk4OXNqoD3QAUeRqLcs1aZOK4U1g=="], - "@rollup/rollup-linux-arm64-musl": ["@rollup/rollup-linux-arm64-musl@4.45.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-PmI1vxQetnM58ZmDFl9/Uk2lpBBby6B6rF4muJc65uZbxCs0EA7hhKCk2PKlmZKuyVSHAyIw3+/SiuMLxKxWog=="], + "@rolldown/binding-linux-arm-gnueabihf": ["@rolldown/binding-linux-arm-gnueabihf@1.2.9", "", { "os": "linux", "cpu": "arm" }, "sha512-1RVzG17pxqbTfYLC352JlLt6kKLG+6Hr30n8DlIJqsnV5luUDd2Qdx9Ayw1Cabfyb1K9k0jXEZ7evxkRoT+uiw=="], - "@rollup/rollup-linux-loongarch64-gnu": ["@rollup/rollup-linux-loongarch64-gnu@4.45.1", "", { "os": "linux", "cpu": "none" }, "sha512-9UmI0VzGmNJ28ibHW2GpE2nF0PBQqsyiS4kcJ5vK+wuwGnV5RlqdczVocDSUfGX/Na7/XINRVoUgJyFIgipoRg=="], + "@rolldown/binding-linux-arm64-gnu": ["@rolldown/binding-linux-arm64-gnu@1.2.9", "", { "os": "linux", "cpu": "arm64" }, "sha512-BXqPvZ2drqVD+/Z8UpKwcs4Mp7grM+eGFku4CAEKrEtcbAsUpzREphK1sogCRZGreVPiMkiiBtw0n3TPteuqvw=="], - "@rollup/rollup-linux-powerpc64le-gnu": ["@rollup/rollup-linux-powerpc64le-gnu@4.45.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-7nR2KY8oEOUTD3pBAxIBBbZr0U7U+R9HDTPNy+5nVVHDXI4ikYniH1oxQz9VoB5PbBU1CZuDGHkLJkd3zLMWsg=="], + "@rolldown/binding-linux-arm64-musl": ["@rolldown/binding-linux-arm64-musl@1.2.9", "", { "os": "linux", "cpu": "arm64" }, "sha512-11vWvo8YDwLzukt27J3aYDWU+gg2P7J+ZOmiJ0hkF5BXZDW7pVya7r40MXDy6ya0i9KamoENSVKIugvJNgFXIA=="], - "@rollup/rollup-linux-riscv64-gnu": ["@rollup/rollup-linux-riscv64-gnu@4.45.1", "", { "os": "linux", "cpu": "none" }, "sha512-nlcl3jgUultKROfZijKjRQLUu9Ma0PeNv/VFHkZiKbXTBQXhpytS8CIj5/NfBeECZtY2FJQubm6ltIxm/ftxpw=="], + "@rolldown/binding-linux-ppc64-gnu": ["@rolldown/binding-linux-ppc64-gnu@1.2.9", "", { "os": "linux", "cpu": "ppc64" }, "sha512-a1tijMkdwsIARtc0F39ApURROkf3NwqinI6TOiSSWCTR7dT96dffNvMUtDHnq64wKNTIZOIlzKrFvvFUznJiyw=="], - "@rollup/rollup-linux-riscv64-musl": ["@rollup/rollup-linux-riscv64-musl@4.45.1", "", { "os": "linux", "cpu": "none" }, "sha512-HJV65KLS51rW0VY6rvZkiieiBnurSzpzore1bMKAhunQiECPuxsROvyeaot/tcK3A3aGnI+qTHqisrpSgQrpgA=="], + "@rolldown/binding-linux-s390x-gnu": ["@rolldown/binding-linux-s390x-gnu@1.2.9", "", { "os": "linux", "cpu": "s390x" }, "sha512-x6SQNdAvv4c3hWqTMaWuawzMX9myaCs/yEmlGsxJzkdClnHW7FbrjQuSiRDhuSYzEYoEMhsaJy9qHG/XNemJPQ=="], - "@rollup/rollup-linux-s390x-gnu": ["@rollup/rollup-linux-s390x-gnu@4.45.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-NITBOCv3Qqc6hhwFt7jLV78VEO/il4YcBzoMGGNxznLgRQf43VQDae0aAzKiBeEPIxnDrACiMgbqjuihx08OOw=="], + "@rolldown/binding-linux-x64-gnu": ["@rolldown/binding-linux-x64-gnu@1.2.9", "", { "os": "linux", "cpu": "x64" }, "sha512-9s0AZ8BFK5/n7B/TBoa2yJE3gI3KURrbXcPBlsAsvjU4VeJKgE90y1YtNxyEUIcHPQkg6/yfF3qihUrcM/Kf0Q=="], - "@rollup/rollup-linux-x64-gnu": ["@rollup/rollup-linux-x64-gnu@4.45.1", "", { "os": "linux", "cpu": "x64" }, "sha512-+E/lYl6qu1zqgPEnTrs4WysQtvc/Sh4fC2nByfFExqgYrqkKWp1tWIbe+ELhixnenSpBbLXNi6vbEEJ8M7fiHw=="], + "@rolldown/binding-linux-x64-musl": ["@rolldown/binding-linux-x64-musl@1.2.9", "", { "os": "linux", "cpu": "x64" }, "sha512-P7VWAmV+WdJluH7ovnRGoiv2i8To7GAZ+kGzfGup635cyL7SyYl3lSUaA3Gp5THf0n/Co5EyEqb2zbqq+nMOHQ=="], - "@rollup/rollup-linux-x64-musl": ["@rollup/rollup-linux-x64-musl@4.45.1", "", { "os": "linux", "cpu": "x64" }, "sha512-a6WIAp89p3kpNoYStITT9RbTbTnqarU7D8N8F2CV+4Cl9fwCOZraLVuVFvlpsW0SbIiYtEnhCZBPLoNdRkjQFw=="], + "@rolldown/binding-openharmony-arm64": ["@rolldown/binding-openharmony-arm64@1.2.9", "", { "os": "none", "cpu": "arm64" }, "sha512-1qixtsE4BK8h+yS3BfmZ09UhA7O/N4IACva6YBr7EBvCJraByTuRcgOTaiA62Tm0vey3UcKXLOaoGHtYmNGEVg=="], - "@rollup/rollup-win32-arm64-msvc": ["@rollup/rollup-win32-arm64-msvc@4.45.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-T5Bi/NS3fQiJeYdGvRpTAP5P02kqSOpqiopwhj0uaXB6nzs5JVi2XMJb18JUSKhCOX8+UE1UKQufyD6Or48dJg=="], + "@rolldown/binding-win32-arm64-msvc": ["@rolldown/binding-win32-arm64-msvc@1.2.9", "", { "os": "win32", "cpu": "arm64" }, "sha512-ok8IQjcEPs1AKZfuEUznVBrJw+gK4soq+bx8b1X2XoMqVClarc1q5JDmVtWXY1xfr6ZuHTAsPXHTgTrqKTZeww=="], - "@rollup/rollup-win32-ia32-msvc": ["@rollup/rollup-win32-ia32-msvc@4.45.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-lxV2Pako3ujjuUe9jiU3/s7KSrDfH6IgTSQOnDWr9aJ92YsFd7EurmClK0ly/t8dzMkDtd04g60WX6yl0sGfdw=="], + "@rolldown/binding-win32-x64-msvc": ["@rolldown/binding-win32-x64-msvc@1.2.9", "", { "os": "win32", "cpu": "x64" }, "sha512-Ip2mXoU0hM0boq3Rf+ekuT653OROSo6aSYcPT1VHE4q52KvyxgFkQgrgb/IEsxOuvQ2fZZbs8khJAyCEPM24/g=="], - "@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.45.1", "", { "os": "win32", "cpu": "x64" }, "sha512-M/fKi4sasCdM8i0aWJjCSFm2qEnYRR8AMLG2kxp6wD13+tMGA4Z1tVAuHkNRjud5SW2EM3naLuK35w9twvf6aA=="], + "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.1", "", {}, "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw=="], "@scure/base": ["@scure/base@1.2.6", "", {}, "sha512-g/nm5FgUa//MCj1gV09zTJTaM6KBAHqLN907YVQqf7zC49+DcO4B1so4ZX07Ef10Twr6nuqYEH9GEggFXA4Fmg=="], @@ -153,6 +101,8 @@ "@sinclair/typebox": ["@sinclair/typebox@0.27.8", "", {}, "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA=="], + "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], + "@types/bun": ["@types/bun@1.2.18", "", { "dependencies": { "bun-types": "1.2.18" } }, "sha512-Xf6RaWVheyemaThV0kUfaAUvCNokFr+bH8Jxp+tTZfx7dAPA8z9ePnP9S9+Vspzuxxx9JRAXhnyccRj3GyCMdQ=="], "@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="], @@ -165,17 +115,17 @@ "@types/react": ["@types/react@19.1.8", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-AwAfQ2Wa5bCx9WP8nZL2uMZWod7J7/JSplxbTmBQ5ms6QpqNYm672H0Vu9ZVKVngQ+ii4R/byguVEUZQyeg44g=="], - "@vitest/expect": ["@vitest/expect@3.2.6", "", { "dependencies": { "@types/chai": "^5.2.2", "@vitest/spy": "3.2.6", "@vitest/utils": "3.2.6", "chai": "^5.2.0", "tinyrainbow": "^2.0.0" } }, "sha512-1+7q9BtaKzEmO+fmNT3kYvoNn5Y71XWAx2Q5HRim4tTVRQVRv4uJFAQ5FbK0OPUeNP/WmVCpxYxoJdvuHVjzBQ=="], + "@vitest/expect": ["@vitest/expect@4.1.11", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", "@vitest/spy": "4.1.11", "@vitest/utils": "4.1.11", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" } }, "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw=="], - "@vitest/mocker": ["@vitest/mocker@3.2.6", "", { "dependencies": { "@vitest/spy": "3.2.6", "estree-walker": "^3.0.3", "magic-string": "^0.30.17" }, "peerDependencies": { "msw": "^2.4.9", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" }, "optionalPeers": ["msw", "vite"] }, "sha512-EZOrpDbkKotFAP7wPAQV1UIyoGOk4oX7ynWhBhLB7v+meMHbQhU16oPpIYGTTe4oFlhpryGpgpcZP/sin3hYuw=="], + "@vitest/mocker": ["@vitest/mocker@4.1.11", "", { "dependencies": { "@vitest/spy": "4.1.11", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, "peerDependencies": { "msw": "^2.4.9", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" }, "optionalPeers": ["msw", "vite"] }, "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ=="], - "@vitest/pretty-format": ["@vitest/pretty-format@3.2.7", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA=="], + "@vitest/pretty-format": ["@vitest/pretty-format@4.1.11", "", { "dependencies": { "tinyrainbow": "^3.1.0" } }, "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw=="], - "@vitest/runner": ["@vitest/runner@3.2.6", "", { "dependencies": { "@vitest/utils": "3.2.6", "pathe": "^2.0.3", "strip-literal": "^3.0.0" } }, "sha512-HYcoSj1w5tcgUnzoF0HcyaAQjpA1gj9ftUJ7iSJSuipc02jW9gKkigwZbjFldAfYHA1fa8UZVRftdMY5msWM9Q=="], + "@vitest/runner": ["@vitest/runner@4.1.11", "", { "dependencies": { "@vitest/utils": "4.1.11", "pathe": "^2.0.3" } }, "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw=="], - "@vitest/snapshot": ["@vitest/snapshot@3.2.6", "", { "dependencies": { "@vitest/pretty-format": "3.2.6", "magic-string": "^0.30.17", "pathe": "^2.0.3" } }, "sha512-H+ZjNTWGpObenh0YnlBctAPnJSI20P81PL8BPzWpx54YXLLTm8hEsWawtcYLMrwvpK48hGxLLbCS+1KRXhsKhw=="], + "@vitest/snapshot": ["@vitest/snapshot@4.1.11", "", { "dependencies": { "@vitest/pretty-format": "4.1.11", "@vitest/utils": "4.1.11", "magic-string": "^0.30.21", "pathe": "^2.0.3" } }, "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog=="], - "@vitest/spy": ["@vitest/spy@3.2.6", "", { "dependencies": { "tinyspy": "^4.0.3" } }, "sha512-oq6BbH68WzcWmwtBrU9nqLeaXTR4XwJF7FSLkKEZo4i6eoXcrxjcwSuTvWBIRUTC6VC72nXYunzqgZA+IKdtxg=="], + "@vitest/spy": ["@vitest/spy@4.1.11", "", {}, "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA=="], "@vitest/ui": ["@vitest/ui@1.6.1", "", { "dependencies": { "@vitest/utils": "1.6.1", "fast-glob": "^3.3.2", "fflate": "^0.8.1", "flatted": "^3.2.9", "pathe": "^1.1.1", "picocolors": "^1.0.0", "sirv": "^2.0.4" }, "peerDependencies": { "vitest": "1.6.1" } }, "sha512-xa57bCPGuzEFqGjPs3vVLyqareG8DX0uMkr5U/v5vLv5/ZUrBrPL7gzxzTJedEyZxFMfsozwTIbbYfEQVo3kgg=="], @@ -191,23 +141,17 @@ "bun-types": ["bun-types@1.2.18", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-04+Eha5NP7Z0A9YgDAzMk5PHR16ZuLVa83b26kH5+cp1qZW4F6FmAURngE7INf4tKOvCE69vYvDEwoNl1tGiWw=="], - "cac": ["cac@6.7.14", "", {}, "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ=="], - - "chai": ["chai@5.3.3", "", { "dependencies": { "assertion-error": "^2.0.1", "check-error": "^2.1.1", "deep-eql": "^5.0.1", "loupe": "^3.1.0", "pathval": "^2.0.0" } }, "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw=="], + "chai": ["chai@6.2.2", "", {}, "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg=="], - "check-error": ["check-error@2.1.3", "", {}, "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA=="], + "convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="], "csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="], - "debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="], - - "deep-eql": ["deep-eql@5.0.2", "", {}, "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q=="], + "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], "diff-sequences": ["diff-sequences@29.6.3", "", {}, "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q=="], - "es-module-lexer": ["es-module-lexer@1.7.0", "", {}, "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA=="], - - "esbuild": ["esbuild@0.21.5", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.21.5", "@esbuild/android-arm": "0.21.5", "@esbuild/android-arm64": "0.21.5", "@esbuild/android-x64": "0.21.5", "@esbuild/darwin-arm64": "0.21.5", "@esbuild/darwin-x64": "0.21.5", "@esbuild/freebsd-arm64": "0.21.5", "@esbuild/freebsd-x64": "0.21.5", "@esbuild/linux-arm": "0.21.5", "@esbuild/linux-arm64": "0.21.5", "@esbuild/linux-ia32": "0.21.5", "@esbuild/linux-loong64": "0.21.5", "@esbuild/linux-mips64el": "0.21.5", "@esbuild/linux-ppc64": "0.21.5", "@esbuild/linux-riscv64": "0.21.5", "@esbuild/linux-s390x": "0.21.5", "@esbuild/linux-x64": "0.21.5", "@esbuild/netbsd-x64": "0.21.5", "@esbuild/openbsd-x64": "0.21.5", "@esbuild/sunos-x64": "0.21.5", "@esbuild/win32-arm64": "0.21.5", "@esbuild/win32-ia32": "0.21.5", "@esbuild/win32-x64": "0.21.5" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw=="], + "es-module-lexer": ["es-module-lexer@2.3.2", "", {}, "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw=="], "estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], @@ -233,7 +177,7 @@ "glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], - "hono": ["hono@4.12.34", "", {}, "sha512-GqXJqY/xJkJmuloTrnV1ZEXG3fqte+VjkUqoRNZXcrUidiUOP4fMSIHHY4tsqZBK++kVyWmt/AAfSUuy57/eSA=="], + "hono": ["hono@4.13.5", "", {}, "sha512-O6+/eCYRkzzzy0rPWwKLiGBR1nFuUPZynnwjxN1MBA62NNqbT0wQEzQyK2gSO5yDIDB336sXQleAhOHrzlYyKw=="], "is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="], @@ -243,11 +187,33 @@ "isows": ["isows@1.0.7", "", { "peerDependencies": { "ws": "*" } }, "sha512-I1fSfDCZL5P0v33sVqeTDSpcstAg/N+wF5HS033mogOVIp4B+oHC7oOCsA3axAbBSGTJ8QubbNmnIRN/h8U7hg=="], - "js-tokens": ["js-tokens@9.0.1", "", {}, "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ=="], + "lightningcss": ["lightningcss@1.33.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.33.0", "lightningcss-darwin-arm64": "1.33.0", "lightningcss-darwin-x64": "1.33.0", "lightningcss-freebsd-x64": "1.33.0", "lightningcss-linux-arm-gnueabihf": "1.33.0", "lightningcss-linux-arm64-gnu": "1.33.0", "lightningcss-linux-arm64-musl": "1.33.0", "lightningcss-linux-x64-gnu": "1.33.0", "lightningcss-linux-x64-musl": "1.33.0", "lightningcss-win32-arm64-msvc": "1.33.0", "lightningcss-win32-x64-msvc": "1.33.0" } }, "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA=="], + + "lightningcss-android-arm64": ["lightningcss-android-arm64@1.33.0", "", { "os": "android", "cpu": "arm64" }, "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg=="], + + "lightningcss-darwin-arm64": ["lightningcss-darwin-arm64@1.33.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg=="], + + "lightningcss-darwin-x64": ["lightningcss-darwin-x64@1.33.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ=="], + + "lightningcss-freebsd-x64": ["lightningcss-freebsd-x64@1.33.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg=="], + + "lightningcss-linux-arm-gnueabihf": ["lightningcss-linux-arm-gnueabihf@1.33.0", "", { "os": "linux", "cpu": "arm" }, "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ=="], + + "lightningcss-linux-arm64-gnu": ["lightningcss-linux-arm64-gnu@1.33.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg=="], + + "lightningcss-linux-arm64-musl": ["lightningcss-linux-arm64-musl@1.33.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ=="], + + "lightningcss-linux-x64-gnu": ["lightningcss-linux-x64-gnu@1.33.0", "", { "os": "linux", "cpu": "x64" }, "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg=="], + + "lightningcss-linux-x64-musl": ["lightningcss-linux-x64-musl@1.33.0", "", { "os": "linux", "cpu": "x64" }, "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw=="], + + "lightningcss-win32-arm64-msvc": ["lightningcss-win32-arm64-msvc@1.33.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA=="], + + "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.33.0", "", { "os": "win32", "cpu": "x64" }, "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA=="], "loupe": ["loupe@2.3.7", "", { "dependencies": { "get-func-name": "^2.0.1" } }, "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA=="], - "magic-string": ["magic-string@0.30.17", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" } }, "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA=="], + "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], @@ -255,9 +221,9 @@ "mrmime": ["mrmime@2.0.1", "", {}, "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ=="], - "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "nanoid": ["nanoid@3.3.19", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug=="], - "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], + "obug": ["obug@2.2.1", "", {}, "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q=="], "openapi-types": ["openapi-types@12.1.3", "", {}, "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw=="], @@ -267,13 +233,11 @@ "pathe": ["pathe@1.1.2", "", {}, "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ=="], - "pathval": ["pathval@2.0.1", "", {}, "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ=="], - "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], "picomatch": ["picomatch@4.0.5", "", {}, "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A=="], - "postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="], + "postcss": ["postcss@8.5.28", "", { "dependencies": { "nanoid": "^3.3.18", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A=="], "pretty-format": ["pretty-format@29.7.0", "", { "dependencies": { "@jest/schemas": "^29.6.3", "ansi-styles": "^5.0.0", "react-is": "^18.0.0" } }, "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ=="], @@ -283,7 +247,7 @@ "reusify": ["reusify@1.1.0", "", {}, "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw=="], - "rollup": ["rollup@4.45.1", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.45.1", "@rollup/rollup-android-arm64": "4.45.1", "@rollup/rollup-darwin-arm64": "4.45.1", "@rollup/rollup-darwin-x64": "4.45.1", "@rollup/rollup-freebsd-arm64": "4.45.1", "@rollup/rollup-freebsd-x64": "4.45.1", "@rollup/rollup-linux-arm-gnueabihf": "4.45.1", "@rollup/rollup-linux-arm-musleabihf": "4.45.1", "@rollup/rollup-linux-arm64-gnu": "4.45.1", "@rollup/rollup-linux-arm64-musl": "4.45.1", "@rollup/rollup-linux-loongarch64-gnu": "4.45.1", "@rollup/rollup-linux-powerpc64le-gnu": "4.45.1", "@rollup/rollup-linux-riscv64-gnu": "4.45.1", "@rollup/rollup-linux-riscv64-musl": "4.45.1", "@rollup/rollup-linux-s390x-gnu": "4.45.1", "@rollup/rollup-linux-x64-gnu": "4.45.1", "@rollup/rollup-linux-x64-musl": "4.45.1", "@rollup/rollup-win32-arm64-msvc": "4.45.1", "@rollup/rollup-win32-ia32-msvc": "4.45.1", "@rollup/rollup-win32-x64-msvc": "4.45.1", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-4iya7Jb76fVpQyLoiVpzUrsjQ12r3dM7fIVz+4NwoYvZOShknRmiv+iu9CClZml5ZLGb0XMcYLutK6w9tgxHDw=="], + "rolldown": ["rolldown@1.2.9", "", { "dependencies": { "@oxc-project/types": "=0.150.0", "@rolldown/pluginutils": "^1.0.0" }, "optionalDependencies": { "@rolldown/binding-android-arm-eabi": "1.2.9", "@rolldown/binding-android-arm64": "1.2.9", "@rolldown/binding-darwin-arm64": "1.2.9", "@rolldown/binding-darwin-x64": "1.2.9", "@rolldown/binding-freebsd-x64": "1.2.9", "@rolldown/binding-linux-arm-gnueabihf": "1.2.9", "@rolldown/binding-linux-arm64-gnu": "1.2.9", "@rolldown/binding-linux-arm64-musl": "1.2.9", "@rolldown/binding-linux-ppc64-gnu": "1.2.9", "@rolldown/binding-linux-s390x-gnu": "1.2.9", "@rolldown/binding-linux-x64-gnu": "1.2.9", "@rolldown/binding-linux-x64-musl": "1.2.9", "@rolldown/binding-openharmony-arm64": "1.2.9", "@rolldown/binding-win32-arm64-msvc": "1.2.9", "@rolldown/binding-win32-x64-msvc": "1.2.9" }, "bin": { "rolldown": "./bin/cli.mjs" } }, "sha512-hx/Pv0N1haXRb11qkfnK5MXB/iqr7i0yjWQqmO9uHqZpBgQSqzc8UsSnEpalsh+j1I8qQ2CkXAkJC8Br3dKSlg=="], "run-parallel": ["run-parallel@1.2.0", "", { "dependencies": { "queue-microtask": "^1.2.2" } }, "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA=="], @@ -295,21 +259,15 @@ "stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="], - "std-env": ["std-env@3.9.0", "", {}, "sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw=="], - - "strip-literal": ["strip-literal@3.1.0", "", { "dependencies": { "js-tokens": "^9.0.1" } }, "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg=="], + "std-env": ["std-env@4.2.0", "", {}, "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw=="], "tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="], - "tinyexec": ["tinyexec@0.3.2", "", {}, "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA=="], + "tinyexec": ["tinyexec@1.3.1", "", {}, "sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA=="], "tinyglobby": ["tinyglobby@0.2.17", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="], - "tinypool": ["tinypool@1.1.1", "", {}, "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg=="], - - "tinyrainbow": ["tinyrainbow@2.0.0", "", {}, "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw=="], - - "tinyspy": ["tinyspy@4.0.4", "", {}, "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q=="], + "tinyrainbow": ["tinyrainbow@3.1.1", "", {}, "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw=="], "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], @@ -321,11 +279,9 @@ "viem": ["viem@2.31.7", "", { "dependencies": { "@noble/curves": "1.9.2", "@noble/hashes": "1.8.0", "@scure/bip32": "1.7.0", "@scure/bip39": "1.6.0", "abitype": "1.0.8", "isows": "1.0.7", "ox": "0.8.1", "ws": "8.18.2" }, "peerDependencies": { "typescript": ">=5.0.4" }, "optionalPeers": ["typescript"] }, "sha512-mpB8Hp6xK77E/b/yJmpAIQcxcOfpbrwWNItjnXaIA8lxZYt4JS433Pge2gg6Hp3PwyFtaUMh01j5L8EXnLTjQQ=="], - "vite": ["vite@5.4.19", "", { "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", "rollup": "^4.20.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || >=20.0.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" }, "optionalPeers": ["@types/node", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser"], "bin": { "vite": "bin/vite.js" } }, "sha512-qO3aKv3HoQC8QKiNSTuUM1l9o/XX3+c+VTgLHbJWHZGeTPVAg2XwazI9UWzoxjIJCGCV2zU60uqMzjeLZuULqA=="], - - "vite-node": ["vite-node@3.2.4", "", { "dependencies": { "cac": "^6.7.14", "debug": "^4.4.1", "es-module-lexer": "^1.7.0", "pathe": "^2.0.3", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" }, "bin": { "vite-node": "vite-node.mjs" } }, "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg=="], + "vite": ["vite@8.3.0", "", { "dependencies": { "lightningcss": "^1.33.0", "picomatch": "^4.0.7", "postcss": "^8.5.28", "rolldown": "~1.2.6", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.7.1", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-lhZBVvEHefgE+HQZC9O7EBJgCU/nVzFNl7vkS4RE0APtWLP02/8QVIkQtzBxPquh7lq5/78NHipTj7ODQ6XuyQ=="], - "vitest": ["vitest@3.2.6", "", { "dependencies": { "@types/chai": "^5.2.2", "@vitest/expect": "3.2.6", "@vitest/mocker": "3.2.6", "@vitest/pretty-format": "^3.2.6", "@vitest/runner": "3.2.6", "@vitest/snapshot": "3.2.6", "@vitest/spy": "3.2.6", "@vitest/utils": "3.2.6", "chai": "^5.2.0", "debug": "^4.4.1", "expect-type": "^1.2.1", "magic-string": "^0.30.17", "pathe": "^2.0.3", "picomatch": "^4.0.2", "std-env": "^3.9.0", "tinybench": "^2.9.0", "tinyexec": "^0.3.2", "tinyglobby": "^0.2.14", "tinypool": "^1.1.1", "tinyrainbow": "^2.0.0", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", "vite-node": "3.2.4", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@types/debug": "^4.1.12", "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "@vitest/browser": "3.2.6", "@vitest/ui": "3.2.6", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@types/debug", "@types/node", "@vitest/browser", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-xejya+bT/j/+R/AGa1XOfRxLmNUlLtlwjRsFUILF+xHfzElmGcmFydy2gqqIrd62ptIEfwVMofd19uNWD9L7Nw=="], + "vitest": ["vitest@4.1.11", "", { "dependencies": { "@vitest/expect": "4.1.11", "@vitest/mocker": "4.1.11", "@vitest/pretty-format": "4.1.11", "@vitest/runner": "4.1.11", "@vitest/snapshot": "4.1.11", "@vitest/spy": "4.1.11", "@vitest/utils": "4.1.11", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", "obug": "^2.1.1", "pathe": "^2.0.3", "picomatch": "^4.0.3", "std-env": "^4.0.0-rc.1", "tinybench": "^2.9.0", "tinyexec": "^1.0.2", "tinyglobby": "^0.2.15", "tinyrainbow": "^3.1.0", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", "@vitest/browser-playwright": "4.1.11", "@vitest/browser-preview": "4.1.11", "@vitest/browser-webdriverio": "4.1.11", "@vitest/coverage-istanbul": "4.1.11", "@vitest/coverage-v8": "4.1.11", "@vitest/ui": "4.1.11", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@opentelemetry/api", "@types/node", "@vitest/browser-playwright", "@vitest/browser-preview", "@vitest/browser-webdriverio", "@vitest/coverage-istanbul", "@vitest/coverage-v8", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "./vitest.mjs" } }, "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw=="], "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], @@ -333,40 +289,26 @@ "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], - "@vitest/expect/@vitest/utils": ["@vitest/utils@3.2.6", "", { "dependencies": { "@vitest/pretty-format": "3.2.6", "loupe": "^3.1.4", "tinyrainbow": "^2.0.0" } }, "sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg=="], + "@vitest/expect/@vitest/utils": ["@vitest/utils@4.1.11", "", { "dependencies": { "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" } }, "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ=="], - "@vitest/runner/@vitest/utils": ["@vitest/utils@3.2.6", "", { "dependencies": { "@vitest/pretty-format": "3.2.6", "loupe": "^3.1.4", "tinyrainbow": "^2.0.0" } }, "sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg=="], + "@vitest/runner/@vitest/utils": ["@vitest/utils@4.1.11", "", { "dependencies": { "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" } }, "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ=="], "@vitest/runner/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], - "@vitest/snapshot/@vitest/pretty-format": ["@vitest/pretty-format@3.2.6", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA=="], + "@vitest/snapshot/@vitest/utils": ["@vitest/utils@4.1.11", "", { "dependencies": { "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" } }, "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ=="], "@vitest/snapshot/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], "bun-types/@types/node": ["@types/node@22.7.5", "", { "dependencies": { "undici-types": "~6.19.2" } }, "sha512-jML7s2NAzMWc//QSJ1a3prpk78cOPchGvXJsC3C6R6PSMoooztvRVQEz89gmBTBY1SPMaqo5teB4uNHPdetShQ=="], - "chai/loupe": ["loupe@3.2.1", "", {}, "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ=="], - "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], - "vite-node/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + "vite/picomatch": ["picomatch@4.0.7", "", {}, "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA=="], - "vitest/@vitest/utils": ["@vitest/utils@3.2.6", "", { "dependencies": { "@vitest/pretty-format": "3.2.6", "loupe": "^3.1.4", "tinyrainbow": "^2.0.0" } }, "sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg=="], + "vitest/@vitest/utils": ["@vitest/utils@4.1.11", "", { "dependencies": { "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" } }, "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ=="], "vitest/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], - "@vitest/expect/@vitest/utils/@vitest/pretty-format": ["@vitest/pretty-format@3.2.6", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA=="], - - "@vitest/expect/@vitest/utils/loupe": ["loupe@3.2.1", "", {}, "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ=="], - - "@vitest/runner/@vitest/utils/@vitest/pretty-format": ["@vitest/pretty-format@3.2.6", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA=="], - - "@vitest/runner/@vitest/utils/loupe": ["loupe@3.2.1", "", {}, "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ=="], - "bun-types/@types/node/undici-types": ["undici-types@6.19.8", "", {}, "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw=="], - - "vitest/@vitest/utils/@vitest/pretty-format": ["@vitest/pretty-format@3.2.6", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA=="], - - "vitest/@vitest/utils/loupe": ["loupe@3.2.1", "", {}, "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ=="], } } diff --git a/dstack/kms/auth-eth/contracts/DstackApp.sol b/dstack/kms/auth-eth/contracts/DstackApp.sol index 6c27e9ccb..7d90f29ef 100644 --- a/dstack/kms/auth-eth/contracts/DstackApp.sol +++ b/dstack/kms/auth-eth/contracts/DstackApp.sol @@ -145,6 +145,7 @@ contract DstackApp is // Add a compose hash to allowed list function addComposeHash(bytes32 composeHash) external onlyOwner { + require(composeHash != bytes32(0), "invalid compose hash"); allowedComposeHashes[composeHash] = true; emit ComposeHashAdded(composeHash); _emitPolicy("compose-hash", composeHash, true); @@ -173,6 +174,7 @@ contract DstackApp is // Add a device ID to allowed list function addDevice(bytes32 deviceId) external onlyOwner { + require(deviceId != bytes32(0), "invalid device ID"); allowedDeviceIds[deviceId] = true; emit DeviceAdded(deviceId); _emitPolicy("device", deviceId, true); diff --git a/dstack/kms/auth-eth/src/ethereum.ts b/dstack/kms/auth-eth/src/ethereum.ts index 2715d5024..ebd655140 100644 --- a/dstack/kms/auth-eth/src/ethereum.ts +++ b/dstack/kms/auth-eth/src/ethereum.ts @@ -50,14 +50,16 @@ export class EthereumBackend { tcbStatus: bootInfo.tcbStatus, advisoryIds: bootInfo.advisoryIds }; + // Read the whole decision from one block. + const blockTag = await this.provider.getBlockNumber(); let response; if (isKms) { - response = await this.kmsContract.isKmsAllowed(bootInfoStruct); + response = await this.kmsContract.isKmsAllowed(bootInfoStruct, { blockTag }); } else { - response = await this.kmsContract.isAppAllowed(bootInfoStruct); + response = await this.kmsContract.isAppAllowed(bootInfoStruct, { blockTag }); } const [isAllowed, reason] = response; - const gatewayAppId = await this.kmsContract.gatewayAppId(); + const gatewayAppId = await this.kmsContract.gatewayAppId({ blockTag }); return { isAllowed, reason, diff --git a/dstack/kms/auth-eth/test/DstackApp.t.sol b/dstack/kms/auth-eth/test/DstackApp.t.sol index 8ad8050e7..646bd827e 100644 --- a/dstack/kms/auth-eth/test/DstackApp.t.sol +++ b/dstack/kms/auth-eth/test/DstackApp.t.sol @@ -391,4 +391,13 @@ contract DstackAppTest is Test { app.addComposeHash(bytes32(uint256(1))); assertTrue(app.allowedComposeHashes(bytes32(uint256(1)))); } + + function test_AddZeroValuedPolicyEntriesIsRejected() public { + vm.startPrank(owner); + vm.expectRevert("invalid compose hash"); + app.addComposeHash(bytes32(0)); + vm.expectRevert("invalid device ID"); + app.addDevice(bytes32(0)); + vm.stopPrank(); + } } diff --git a/dstack/kms/auth-mock/bun.lock b/dstack/kms/auth-mock/bun.lock index 35fa3ece6..955a0dda3 100644 --- a/dstack/kms/auth-mock/bun.lock +++ b/dstack/kms/auth-mock/bun.lock @@ -1,12 +1,11 @@ { "lockfileVersion": 1, - "configVersion": 0, "workspaces": { "": { "name": "auth-mock", "dependencies": { "@hono/zod-validator": "0.2.2", - "hono": "4.12.34", + "hono": "4.13.5", "zod": "3.25.76", }, "devDependencies": { @@ -16,7 +15,7 @@ "openapi-types": "12.1.3", "oxlint": "0.9.10", "typescript": "5.8.3", - "vitest": "4.1.0", + "vitest": "4.1.11", }, }, }, @@ -111,17 +110,17 @@ "@types/react": ["@types/react@19.1.8", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-AwAfQ2Wa5bCx9WP8nZL2uMZWod7J7/JSplxbTmBQ5ms6QpqNYm672H0Vu9ZVKVngQ+ii4R/byguVEUZQyeg44g=="], - "@vitest/expect": ["@vitest/expect@4.1.0", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", "@vitest/spy": "4.1.0", "@vitest/utils": "4.1.0", "chai": "^6.2.2", "tinyrainbow": "^3.0.3" } }, "sha512-EIxG7k4wlWweuCLG9Y5InKFwpMEOyrMb6ZJ1ihYu02LVj/bzUwn2VMU+13PinsjRW75XnITeFrQBMH5+dLvCDA=="], + "@vitest/expect": ["@vitest/expect@4.1.11", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", "@vitest/spy": "4.1.11", "@vitest/utils": "4.1.11", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" } }, "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw=="], - "@vitest/mocker": ["@vitest/mocker@4.1.0", "", { "dependencies": { "@vitest/spy": "4.1.0", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, "peerDependencies": { "msw": "^2.4.9", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0-0" }, "optionalPeers": ["msw", "vite"] }, "sha512-evxREh+Hork43+Y4IOhTo+h5lGmVRyjqI739Rz4RlUPqwrkFFDF6EMvOOYjTx4E8Tl6gyCLRL8Mu7Ry12a13Tw=="], + "@vitest/mocker": ["@vitest/mocker@4.1.11", "", { "dependencies": { "@vitest/spy": "4.1.11", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, "peerDependencies": { "msw": "^2.4.9", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" }, "optionalPeers": ["msw", "vite"] }, "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ=="], - "@vitest/pretty-format": ["@vitest/pretty-format@4.1.0", "", { "dependencies": { "tinyrainbow": "^3.0.3" } }, "sha512-3RZLZlh88Ib0J7NQTRATfc/3ZPOnSUn2uDBUoGNn5T36+bALixmzphN26OUD3LRXWkJu4H0s5vvUeqBiw+kS0A=="], + "@vitest/pretty-format": ["@vitest/pretty-format@4.1.11", "", { "dependencies": { "tinyrainbow": "^3.1.0" } }, "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw=="], - "@vitest/runner": ["@vitest/runner@4.1.0", "", { "dependencies": { "@vitest/utils": "4.1.0", "pathe": "^2.0.3" } }, "sha512-Duvx2OzQ7d6OjchL+trw+aSrb9idh7pnNfxrklo14p3zmNL4qPCDeIJAK+eBKYjkIwG96Bc6vYuxhqDXQOWpoQ=="], + "@vitest/runner": ["@vitest/runner@4.1.11", "", { "dependencies": { "@vitest/utils": "4.1.11", "pathe": "^2.0.3" } }, "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw=="], - "@vitest/snapshot": ["@vitest/snapshot@4.1.0", "", { "dependencies": { "@vitest/pretty-format": "4.1.0", "@vitest/utils": "4.1.0", "magic-string": "^0.30.21", "pathe": "^2.0.3" } }, "sha512-0Vy9euT1kgsnj1CHttwi9i9o+4rRLEaPRSOJ5gyv579GJkNpgJK+B4HSv/rAWixx2wdAFci1X4CEPjiu2bXIMg=="], + "@vitest/snapshot": ["@vitest/snapshot@4.1.11", "", { "dependencies": { "@vitest/pretty-format": "4.1.11", "@vitest/utils": "4.1.11", "magic-string": "^0.30.21", "pathe": "^2.0.3" } }, "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog=="], - "@vitest/spy": ["@vitest/spy@4.1.0", "", {}, "sha512-pz77k+PgNpyMDv2FV6qmk5ZVau6c3R8HC8v342T2xlFxQKTrSeYw9waIJG8KgV9fFwAtTu4ceRzMivPTH6wSxw=="], + "@vitest/spy": ["@vitest/spy@4.1.11", "", {}, "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA=="], "@vitest/ui": ["@vitest/ui@1.6.1", "", { "dependencies": { "@vitest/utils": "1.6.1", "fast-glob": "^3.3.2", "fflate": "^0.8.1", "flatted": "^3.2.9", "pathe": "^1.1.1", "picocolors": "^1.0.0", "sirv": "^2.0.4" }, "peerDependencies": { "vitest": "1.6.1" } }, "sha512-xa57bCPGuzEFqGjPs3vVLyqareG8DX0uMkr5U/v5vLv5/ZUrBrPL7gzxzTJedEyZxFMfsozwTIbbYfEQVo3kgg=="], @@ -169,7 +168,7 @@ "glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], - "hono": ["hono@4.12.34", "", {}, "sha512-GqXJqY/xJkJmuloTrnV1ZEXG3fqte+VjkUqoRNZXcrUidiUOP4fMSIHHY4tsqZBK++kVyWmt/AAfSUuy57/eSA=="], + "hono": ["hono@4.13.5", "", {}, "sha512-O6+/eCYRkzzzy0rPWwKLiGBR1nFuUPZynnwjxN1MBA62NNqbT0wQEzQyK2gSO5yDIDB336sXQleAhOHrzlYyKw=="], "is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="], @@ -269,19 +268,19 @@ "vite": ["vite@8.1.5", "", { "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.5", "postcss": "^8.5.17", "rolldown": "~1.1.5", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.3.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw=="], - "vitest": ["vitest@4.1.0", "", { "dependencies": { "@vitest/expect": "4.1.0", "@vitest/mocker": "4.1.0", "@vitest/pretty-format": "4.1.0", "@vitest/runner": "4.1.0", "@vitest/snapshot": "4.1.0", "@vitest/spy": "4.1.0", "@vitest/utils": "4.1.0", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", "obug": "^2.1.1", "pathe": "^2.0.3", "picomatch": "^4.0.3", "std-env": "^4.0.0-rc.1", "tinybench": "^2.9.0", "tinyexec": "^1.0.2", "tinyglobby": "^0.2.15", "tinyrainbow": "^3.0.3", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0-0", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", "@vitest/browser-playwright": "4.1.0", "@vitest/browser-preview": "4.1.0", "@vitest/browser-webdriverio": "4.1.0", "@vitest/ui": "4.1.0", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@opentelemetry/api", "@types/node", "@vitest/browser-playwright", "@vitest/browser-preview", "@vitest/browser-webdriverio", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-YbDrMF9jM2Lqc++2530UourxZHmkKLxrs4+mYhEwqWS97WJ7wOYEkcr+QfRgJ3PW9wz3odRijLZjHEaRLTNbqw=="], + "vitest": ["vitest@4.1.11", "", { "dependencies": { "@vitest/expect": "4.1.11", "@vitest/mocker": "4.1.11", "@vitest/pretty-format": "4.1.11", "@vitest/runner": "4.1.11", "@vitest/snapshot": "4.1.11", "@vitest/spy": "4.1.11", "@vitest/utils": "4.1.11", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", "obug": "^2.1.1", "pathe": "^2.0.3", "picomatch": "^4.0.3", "std-env": "^4.0.0-rc.1", "tinybench": "^2.9.0", "tinyexec": "^1.0.2", "tinyglobby": "^0.2.15", "tinyrainbow": "^3.1.0", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", "@vitest/browser-playwright": "4.1.11", "@vitest/browser-preview": "4.1.11", "@vitest/browser-webdriverio": "4.1.11", "@vitest/coverage-istanbul": "4.1.11", "@vitest/coverage-v8": "4.1.11", "@vitest/ui": "4.1.11", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@opentelemetry/api", "@types/node", "@vitest/browser-playwright", "@vitest/browser-preview", "@vitest/browser-webdriverio", "@vitest/coverage-istanbul", "@vitest/coverage-v8", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "./vitest.mjs" } }, "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw=="], "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], - "@vitest/expect/@vitest/utils": ["@vitest/utils@4.1.0", "", { "dependencies": { "@vitest/pretty-format": "4.1.0", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.0.3" } }, "sha512-XfPXT6a8TZY3dcGY8EdwsBulFCIw+BeeX0RZn2x/BtiY/75YGh8FeWGG8QISN/WhaqSrE2OrlDgtF8q5uhOTmw=="], + "@vitest/expect/@vitest/utils": ["@vitest/utils@4.1.11", "", { "dependencies": { "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" } }, "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ=="], - "@vitest/runner/@vitest/utils": ["@vitest/utils@4.1.0", "", { "dependencies": { "@vitest/pretty-format": "4.1.0", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.0.3" } }, "sha512-XfPXT6a8TZY3dcGY8EdwsBulFCIw+BeeX0RZn2x/BtiY/75YGh8FeWGG8QISN/WhaqSrE2OrlDgtF8q5uhOTmw=="], + "@vitest/runner/@vitest/utils": ["@vitest/utils@4.1.11", "", { "dependencies": { "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" } }, "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ=="], "@vitest/runner/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], - "@vitest/snapshot/@vitest/utils": ["@vitest/utils@4.1.0", "", { "dependencies": { "@vitest/pretty-format": "4.1.0", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.0.3" } }, "sha512-XfPXT6a8TZY3dcGY8EdwsBulFCIw+BeeX0RZn2x/BtiY/75YGh8FeWGG8QISN/WhaqSrE2OrlDgtF8q5uhOTmw=="], + "@vitest/snapshot/@vitest/utils": ["@vitest/utils@4.1.11", "", { "dependencies": { "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" } }, "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ=="], "@vitest/snapshot/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], @@ -289,7 +288,7 @@ "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], - "vitest/@vitest/utils": ["@vitest/utils@4.1.0", "", { "dependencies": { "@vitest/pretty-format": "4.1.0", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.0.3" } }, "sha512-XfPXT6a8TZY3dcGY8EdwsBulFCIw+BeeX0RZn2x/BtiY/75YGh8FeWGG8QISN/WhaqSrE2OrlDgtF8q5uhOTmw=="], + "vitest/@vitest/utils": ["@vitest/utils@4.1.11", "", { "dependencies": { "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" } }, "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ=="], "vitest/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], diff --git a/dstack/kms/auth-simple/bun.lock b/dstack/kms/auth-simple/bun.lock index dc65b4382..211959aa6 100644 --- a/dstack/kms/auth-simple/bun.lock +++ b/dstack/kms/auth-simple/bun.lock @@ -5,7 +5,7 @@ "name": "auth-simple", "dependencies": { "@hono/zod-validator": "0.2.2", - "hono": "4.12.34", + "hono": "4.13.5", "zod": "3.25.76", }, "devDependencies": { @@ -218,7 +218,7 @@ "glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], - "hono": ["hono@4.12.34", "", {}, "sha512-GqXJqY/xJkJmuloTrnV1ZEXG3fqte+VjkUqoRNZXcrUidiUOP4fMSIHHY4tsqZBK++kVyWmt/AAfSUuy57/eSA=="], + "hono": ["hono@4.13.5", "", {}, "sha512-O6+/eCYRkzzzy0rPWwKLiGBR1nFuUPZynnwjxN1MBA62NNqbT0wQEzQyK2gSO5yDIDB336sXQleAhOHrzlYyKw=="], "is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="], diff --git a/dstack/kms/dstack-app/builder/README.md b/dstack/kms/dstack-app/builder/README.md index 923b34ebc..382197e5d 100644 --- a/dstack/kms/dstack-app/builder/README.md +++ b/dstack/kms/dstack-app/builder/README.md @@ -58,17 +58,17 @@ GIT_REV=HEAD \ IMAGE_VERSION=0.6.0 \ IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack \ OCI_TAR=/tmp/kms.oci.tar \ - ./build-image.sh dstacktee/dstack-kms:0.6.0 + ./build-image.sh ghcr.io/dstack-tee/dstack-kms:0.6.0 python3 -c 'import json,tarfile;t=tarfile.open("/tmp/kms.oci.tar");print(json.load(t.extractfile("index.json"))["manifests"][0]["digest"])' ``` `IMAGE_VERSION` is part of the image metadata, so it must match the release for the digests to match. The printed digest is what the registry reports for -`dstacktee/dstack-kms:0.6.0`; compare it with: +`ghcr.io/dstack-tee/dstack-kms:0.6.0`; compare it with: ```bash -docker buildx imagetools inspect dstacktee/dstack-kms:0.6.0 --format '{{.Manifest.Digest}}' +docker buildx imagetools inspect ghcr.io/dstack-tee/dstack-kms:0.6.0 --format '{{.Manifest.Digest}}' ``` This is the digest that `deploy-to-vmm.sh` pins in `KMS_IMAGE`, and that in turn diff --git a/dstack/kms/kms.toml b/dstack/kms/kms.toml index 5beb28d80..3a6ad527e 100644 --- a/dstack/kms/kms.toml +++ b/dstack/kms/kms.toml @@ -45,9 +45,18 @@ sev_snp_key_release = false # AWS EC2 NitroTPM key/cert release remains disabled unless this local KMS gate # is explicitly enabled. NitroTPM is a new, non-confidential-compute attestation # mode (the AWS hypervisor is in the TCB), so it is opt-in like SEV-SNP. External -# auth policy must still allow the verified BootInfo. +# auth policy must still allow the verified BootInfo. NitroTPM has no TCB status, +# so the verifier reports "UpToDate" and requireTcbUpToDate does not gate it. aws_nitro_tpm_key_release = false +# AWS Nitro Enclave key/cert release remains disabled unless this local KMS gate +# is explicitly enabled. The signed NSM document measures the enclave image, but +# the AWS hypervisor is in the TCB and dstack extends no runtime measurement +# register on this platform, so app_id and instance_id are read from an event +# list nothing replays against hardware. Opt-in like SEV-SNP and NitroTPM. +# External auth policy must still allow the verified BootInfo. +nitro_enclave_key_release = false + # Optional attestation trust-anchor files. Omitted entries use vendor roots. [core.attestation] insecure_allow_external_trust_anchors = false diff --git a/dstack/kms/src/config.rs b/dstack/kms/src/config.rs index ba3f9f93e..b479d0c50 100644 --- a/dstack/kms/src/config.rs +++ b/dstack/kms/src/config.rs @@ -53,6 +53,17 @@ pub(crate) struct KmsConfig { /// mirroring `sev_snp_key_release`. #[serde(default)] pub aws_nitro_tpm_key_release: bool, + /// Whether to enable the additional local release gate for AWS Nitro Enclave + /// key/cert material. The NSM document measures the enclave image (PCR0/1/2), + /// which is what `os_image_hash` and `compose_hash` are built from, but the + /// AWS hypervisor is in the TCB and dstack extends no runtime measurement + /// register on this platform — so `app_id` and `instance_id` are read from an + /// event list nothing replays against hardware. Production deployments + /// therefore need an explicit KMS opt-in as well as a successful external + /// policy decision — mirroring `sev_snp_key_release` and + /// `aws_nitro_tpm_key_release`. + #[serde(default)] + pub nitro_enclave_key_release: bool, #[serde(default)] pub site_name: String, /// Whether the KMS embeds an attestation in its own RPC certificate. @@ -112,6 +123,13 @@ fn default_true() -> bool { } impl KmsConfig { + /// The root keys are the last files a bootstrap writes, so both existing + /// marks a completed one. Anything short of that is left over from an + /// interrupted bootstrap and may be overwritten. + pub fn root_keys_exist(&self) -> bool { + self.root_ca_key().exists() && self.k256_key().exists() + } + pub fn keys_exists(&self) -> bool { self.tmp_ca_cert().exists() && self.tmp_ca_key().exists() diff --git a/dstack/kms/src/main_service.rs b/dstack/kms/src/main_service.rs index 726a0a341..5e9bc3ffd 100644 --- a/dstack/kms/src/main_service.rs +++ b/dstack/kms/src/main_service.rs @@ -8,9 +8,10 @@ use std::{ atomic::{AtomicU64, Ordering}, Arc, }, + time::Duration, }; -use anyhow::{bail, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use dstack_kms_rpc::{ kms_server::{KmsRpc, KmsServer}, AppId, AppKeyResponse, GetAppKeyRequest, GetKmsKeyRequest, GetMetaResponse, @@ -20,6 +21,7 @@ use dstack_kms_rpc::{ use dstack_verifier::{CvmVerifier, VerificationDetails}; use fs_err as fs; use k256::ecdsa::SigningKey; +use moka::future::Cache; use ra_rpc::{CallContext, RpcCall}; use ra_tls::{ attestation::{AttestationVerifier, TeeVariant, VerifiedAttestation}, @@ -29,7 +31,9 @@ use ra_tls::{ use scale::Decode; use tokio::sync::OnceCell; use tracing::{info, warn}; -use upgrade_authority::{build_boot_info, ensure_app_id_len, local_kms_boot_info, BootInfo}; +use upgrade_authority::{ + build_boot_info, ensure_app_id_len, local_kms_boot_info, BootInfo, GetInfoResponse, +}; use crate::{ config::KmsConfig, @@ -39,6 +43,13 @@ use crate::{ pub(crate) mod amd_attest; pub(crate) mod upgrade_authority; +/// How long the auth API's info and this KMS's own authorization are reused. +/// +/// `GetMeta`, `GetAppEnvEncryptPubKey` and `GetTempCaCert` need no client certificate, and each +/// would otherwise cost the auth API a round of chain RPC calls. Concurrent callers share one +/// in-flight request. App authorization is never cached. +const AUTH_API_CACHE_TTL: Duration = Duration::from_secs(1); + #[derive(Clone)] pub struct KmsState { inner: Arc, @@ -61,6 +72,8 @@ pub struct KmsStateInner { verifier: CvmVerifier, attestation_verifier: Arc, self_boot_info: OnceCell, + self_allowed: Cache<(), ()>, + auth_api_info: Cache<(), GetInfoResponse>, metrics: KmsMetrics, } @@ -95,6 +108,13 @@ impl KmsMetrics { } } +fn short_lived_cache() -> Cache<(), V> { + Cache::builder() + .max_capacity(1) + .time_to_live(AUTH_API_CACHE_TTL) + .build() +} + /// remove a single cache entry (a hex-named subdir/file) under `parent_dir`, or /// everything when `sub_dir == "all"`. A non-hex key is rejected to keep the /// deletion confined to the cache. @@ -168,6 +188,8 @@ impl KmsState { verifier, attestation_verifier, self_boot_info: OnceCell::new(), + self_allowed: short_lived_cache(), + auth_api_info: short_lived_cache(), metrics: KmsMetrics::default(), }), }) @@ -211,29 +233,48 @@ pub(crate) fn build_boot_info_for_attestation( build_boot_info(att, use_boottime_mr, vm_config_str) } -fn ensure_key_release_allowed( - boot_info: &BootInfo, - snp_enabled: bool, - aws_nitro_tpm_enabled: bool, -) -> Result<()> { +/// The per-platform local key-release opt-ins, read from `KmsConfig`. +/// +/// Carried as a named struct rather than positional bools so that adding a +/// platform does not silently reorder an existing call site. +#[derive(Debug, Clone, Copy, Default)] +struct KeyReleasePolicy { + snp: bool, + aws_nitro_tpm: bool, + nitro_enclave: bool, +} + +impl From<&KmsConfig> for KeyReleasePolicy { + fn from(config: &KmsConfig) -> Self { + Self { + snp: config.sev_snp_key_release, + aws_nitro_tpm: config.aws_nitro_tpm_key_release, + nitro_enclave: config.nitro_enclave_key_release, + } + } +} + +fn ensure_key_release_allowed(boot_info: &BootInfo, policy: KeyReleasePolicy) -> Result<()> { match boot_info.tee_variant { - TeeVariant::DstackAmdSevSnp if !snp_enabled => { + TeeVariant::DstackAmdSevSnp if !policy.snp => { bail!("amd sev-snp key release is not enabled") } - TeeVariant::DstackAwsNitroTpm if !aws_nitro_tpm_enabled => { + TeeVariant::DstackAwsNitroTpm if !policy.aws_nitro_tpm => { bail!("aws nitro-tpm key release is not enabled") } + TeeVariant::DstackNitroEnclave if !policy.nitro_enclave => { + bail!("aws nitro enclave key release is not enabled") + } _ => Ok(()), } } fn ensure_self_key_release_allowed( self_boot_info: Option<&BootInfo>, - snp_enabled: bool, - aws_nitro_tpm_enabled: bool, + policy: KeyReleasePolicy, ) -> Result<()> { if let Some(boot_info) = self_boot_info { - ensure_key_release_allowed(boot_info, snp_enabled, aws_nitro_tpm_enabled)?; + ensure_key_release_allowed(boot_info, policy)?; } Ok(()) } @@ -249,16 +290,23 @@ impl RpcHandler { .get_or_try_init(|| local_kms_boot_info(&self.state.attestation_verifier)) .await .context("Failed to load cached self boot info")?; - let response = self - .state - .config - .auth_api - .is_app_allowed(boot_info, true) + self.state + .self_allowed + .try_get_with((), async { + let response = self + .state + .config + .auth_api + .is_app_allowed(boot_info, true) + .await + .context("Failed to call self KMS auth check")?; + if !response.is_allowed { + bail!("KMS is not allowed: {}", response.reason); + } + Ok(()) + }) .await - .context("Failed to call self KMS auth check")?; - if !response.is_allowed { - bail!("KMS is not allowed: {}", response.reason); - } + .map_err(|err| anyhow!("{err:#}"))?; Ok(Some(boot_info)) } @@ -367,11 +415,7 @@ impl KmsRpc for RpcHandler { .ensure_app_boot_allowed(&request.vm_config) .await .context("App not allowed")?; - ensure_key_release_allowed( - &boot_info, - self.state.config.sev_snp_key_release, - self.state.config.aws_nitro_tpm_key_release, - )?; + ensure_key_release_allowed(&boot_info, (&self.state.config).into())?; let app_id = boot_info.app_id; let instance_id = boot_info.instance_id; let os_image_hash = boot_info.os_image_hash; @@ -455,7 +499,12 @@ impl KmsRpc for RpcHandler { let bootstrap_info = fs::read_to_string(self.state.config.bootstrap_info()) .ok() .and_then(|s| serde_json::from_str(&s).ok()); - let info = self.state.config.auth_api.get_info().await?; + let info = self + .state + .auth_api_info + .try_get_with((), self.state.config.auth_api.get_info()) + .await + .map_err(|err| anyhow!("{err:#}"))?; Ok(GetMetaResponse { ca_cert: self.state.inner.root_ca.pem_cert.clone(), allow_any_upgrade: self.state.inner.config.auth_api.is_dev(), @@ -480,11 +529,7 @@ impl KmsRpc for RpcHandler { .await .context("KMS self authorization failed")?; let info = self.ensure_kms_allowed(&request.vm_config).await?; - ensure_key_release_allowed( - &info, - self.state.config.sev_snp_key_release, - self.state.config.aws_nitro_tpm_key_release, - )?; + ensure_key_release_allowed(&info, (&self.state.config).into())?; Ok(KmsKeyResponse { temp_ca_key: self.state.inner.temp_ca_key.clone(), keys: vec![KmsKeys { @@ -513,11 +558,7 @@ impl KmsRpc for RpcHandler { .ensure_self_allowed() .await .context("KMS self authorization failed")?; - ensure_self_key_release_allowed( - self_boot_info, - self.state.config.sev_snp_key_release, - self.state.config.aws_nitro_tpm_key_release, - )?; + ensure_self_key_release_allowed(self_boot_info, (&self.state.config).into())?; Ok(GetTempCaCertResponse { temp_ca_cert: self.state.inner.temp_ca_cert.clone(), temp_ca_key: self.state.inner.temp_ca_key.clone(), @@ -556,11 +597,7 @@ impl KmsRpc for RpcHandler { let app_info = self .ensure_app_attestation_allowed(&attestation, false, true, &request.vm_config) .await?; - ensure_key_release_allowed( - &app_info.boot_info, - self.state.config.sev_snp_key_release, - self.state.config.aws_nitro_tpm_key_release, - )?; + ensure_key_release_allowed(&app_info.boot_info, (&self.state.config).into())?; let app_ca = self.derive_app_ca(&app_info.boot_info.app_id)?; let cert = app_ca .sign_csr(&csr, Some(&app_info.boot_info.app_id), "app:custom") @@ -598,6 +635,18 @@ mod tests { }; use cc_eventlog::RuntimeEvent; use sha2::{Digest, Sha256, Sha384}; + use std::time::Duration; + + const SNP_ENABLED: KeyReleasePolicy = KeyReleasePolicy { + snp: true, + aws_nitro_tpm: false, + nitro_enclave: false, + }; + const NITRO_ENCLAVE_ENABLED: KeyReleasePolicy = KeyReleasePolicy { + snp: false, + aws_nitro_tpm: false, + nitro_enclave: true, + }; #[test] fn remove_cache_only_deletes_the_named_hex_entry() { @@ -938,14 +987,21 @@ mod tests { .expect("aws nitrotpm attestation should produce KMS boot info"); // disabled (the default) fails closed for the new AWS NitroTPM mode - let err = ensure_key_release_allowed(&boot_info, false, false).unwrap_err(); + let err = ensure_key_release_allowed(&boot_info, KeyReleasePolicy::default()).unwrap_err(); assert!(err.to_string().contains("not enabled")); // explicitly enabled permits it - ensure_key_release_allowed(&boot_info, false, true).unwrap(); + ensure_key_release_allowed( + &boot_info, + KeyReleasePolicy { + aws_nitro_tpm: true, + ..Default::default() + }, + ) + .unwrap(); // A TDX boot info is unaffected by the AWS gate even when it is disabled. boot_info.tee_variant = TeeVariant::DstackTdx; - ensure_key_release_allowed(&boot_info, false, false).unwrap(); + ensure_key_release_allowed(&boot_info, KeyReleasePolicy::default()).unwrap(); } #[test] @@ -1090,9 +1146,8 @@ mod tests { #[test] fn snp_key_release_requires_explicit_enablement() { let boot_info = snp_boot_info(); - let enabled = false; - let err = ensure_key_release_allowed(&boot_info, enabled, false) + let err = ensure_key_release_allowed(&boot_info, KeyReleasePolicy::default()) .expect_err("snp boot info must not be key-release enabled by default"); assert!( err.to_string() @@ -1104,32 +1159,206 @@ mod tests { #[test] fn snp_key_release_accepts_auth_approved_boot_info_when_enabled() { let boot_info = snp_boot_info(); - let enabled = true; - ensure_key_release_allowed(&boot_info, enabled, false) + ensure_key_release_allowed(&boot_info, SNP_ENABLED) .expect("explicitly enabled SNP key release should allow auth-approved boot info"); } #[test] fn snp_key_release_leaves_tcb_and_advisory_policy_to_auth_api() { let mut boot_info = snp_boot_info(); - let enabled = true; boot_info.tcb_status = "OutOfDate".to_string(); boot_info.advisory_ids.push("SNP-TEST-ADVISORY".to_string()); - ensure_key_release_allowed(&boot_info, enabled, false) + ensure_key_release_allowed(&boot_info, SNP_ENABLED) .expect("TCB/advisory policy should be decided by the auth API, not this local gate"); } #[test] fn snp_self_boot_info_uses_same_release_policy_for_temp_ca() { let boot_info = snp_boot_info(); - let disabled = false; - let enabled = true; - ensure_self_key_release_allowed(Some(&boot_info), disabled, false) + ensure_self_key_release_allowed(Some(&boot_info), KeyReleasePolicy::default()) .expect_err("disabled SNP self boot info must not receive temp CA key material"); - ensure_self_key_release_allowed(Some(&boot_info), enabled, false) + ensure_self_key_release_allowed(Some(&boot_info), SNP_ENABLED) .expect("enabled clean SNP self boot info should pass the temp CA release gate"); } + + #[test] + fn nitro_enclave_key_release_requires_explicit_enablement() { + let mut boot_info = snp_boot_info(); + boot_info.tee_variant = TeeVariant::DstackNitroEnclave; + + // disabled (the default) fails closed + let err = ensure_key_release_allowed(&boot_info, KeyReleasePolicy::default()) + .expect_err("nitro enclave boot info must not be key-release enabled by default"); + assert!( + err.to_string() + .contains("aws nitro enclave key release is not enabled"), + "unexpected error: {err:?}" + ); + + // explicitly enabled permits it + ensure_key_release_allowed(&boot_info, NITRO_ENCLAVE_ENABLED) + .expect("explicitly enabled nitro enclave key release should allow the boot info"); + + // the temp CA path uses the same gate + ensure_self_key_release_allowed(Some(&boot_info), KeyReleasePolicy::default()).expect_err( + "disabled nitro enclave self boot info must not receive temp CA key material", + ); + ensure_self_key_release_allowed(Some(&boot_info), NITRO_ENCLAVE_ENABLED) + .expect("enabled nitro enclave self boot info should pass the temp CA release gate"); + } + + #[test] + fn key_release_gates_do_not_leak_across_platforms() { + let mut boot_info = snp_boot_info(); + + // Enabling one platform must not enable another. + boot_info.tee_variant = TeeVariant::DstackNitroEnclave; + ensure_key_release_allowed(&boot_info, SNP_ENABLED) + .expect_err("the SNP opt-in must not release keys to a nitro enclave"); + boot_info.tee_variant = TeeVariant::DstackAmdSevSnp; + ensure_key_release_allowed(&boot_info, NITRO_ENCLAVE_ENABLED) + .expect_err("the nitro enclave opt-in must not release keys to an SNP guest"); + + // TDX and GCP TDX have no local gate and are unaffected by all of them. + for variant in [TeeVariant::DstackTdx, TeeVariant::DstackGcpTdx] { + boot_info.tee_variant = variant; + ensure_key_release_allowed(&boot_info, KeyReleasePolicy::default()) + .expect("TDX key release has no local opt-in gate"); + } + } + + /// An auth API that counts requests and answers them all at once, so a + /// test can tell how many upstream calls a burst of concurrent RPCs cost. + fn serve_concurrent_auth_api( + body: &'static str, + ) -> (String, std::sync::Arc) { + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let requests = Arc::new(AtomicUsize::new(0)); + let counter = requests.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + counter.fetch_add(1, Ordering::SeqCst); + std::thread::spawn(move || { + let mut chunk = [0u8; 4096]; + let _ = stream.read(&mut chunk); + // Hold the response long enough that a burst of callers is + // genuinely in flight together rather than serialized. + std::thread::sleep(Duration::from_millis(200)); + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + }); + } + }); + (format!("http://{address}"), requests) + } + + /// A `KmsState` backed by freshly generated root material in `cert_dir`, + /// with its auth API pointed at `webhook_url`. + fn kms_state(cert_dir: &Path, webhook_url: &str) -> KmsState { + use ra_tls::rcgen::{KeyPair, PKCS_ECDSA_P256_SHA256}; + use rocket::figment::{ + providers::{Format, Toml}, + Figment, + }; + + let ca_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap(); + let ca_cert = CertRequest::builder() + .org_name("Dstack") + .subject("Dstack KMS CA") + .ca_level(1) + .key(&ca_key) + .build() + .self_signed() + .unwrap(); + let tmp_ca_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap(); + let tmp_ca_cert = CertRequest::builder() + .org_name("Dstack") + .subject("Dstack Client Temp CA") + .ca_level(0) + .key(&tmp_ca_key) + .build() + .self_signed() + .unwrap(); + fs::write(cert_dir.join("root-ca.key"), ca_key.serialize_pem()).unwrap(); + fs::write(cert_dir.join("root-ca.crt"), ca_cert.pem()).unwrap(); + fs::write(cert_dir.join("tmp-ca.key"), tmp_ca_key.serialize_pem()).unwrap(); + fs::write(cert_dir.join("tmp-ca.crt"), tmp_ca_cert.pem()).unwrap(); + fs::write( + cert_dir.join("root-k256.key"), + SigningKey::random(&mut rand::rngs::OsRng).to_bytes(), + ) + .unwrap(); + + let overrides = format!( + r#" + [core] + cert_dir = "{}" + attest_rpc_cert = false + enforce_self_authorization = false + + [core.image] + cache_dir = "{}" + + [core.auth_api] + type = "webhook" + + [core.auth_api.webhook] + url = "{webhook_url}" + "#, + cert_dir.display(), + cert_dir.display(), + ); + let config: KmsConfig = Figment::from(rocket::Config::default()) + .merge(Toml::string(crate::config::DEFAULT_CONFIG)) + .merge(Toml::string(&overrides)) + .focus("core") + .extract() + .unwrap(); + KmsState::new(config).unwrap() + } + + const AUTH_API_INFO: &str = r#"{"status":"ok","kmsContractAddr":"0xkms","ethRpcUrl":"https://rpc.example","gatewayAppId":"0xgateway","chainId":1,"appImplementation":"0ximpl"}"#; + + #[rocket::async_test] + async fn concurrent_get_meta_calls_share_one_auth_api_request() { + let dir = tempfile::tempdir().unwrap(); + let (url, upstream_requests) = serve_concurrent_auth_api(AUTH_API_INFO); + let state = kms_state(dir.path(), &url); + + let burst: Vec<_> = (0..24) + .map(|_| { + let state = state.clone(); + rocket::tokio::spawn(async move { + RpcHandler { + state, + attestation: None, + } + .get_meta() + .await + }) + }) + .collect(); + for handle in burst { + let meta = handle.await.unwrap().unwrap(); + assert_eq!(meta.chain_id, Some(1)); + assert_eq!(meta.gateway_app_id.as_deref(), Some("0xgateway")); + } + assert_eq!( + upstream_requests.load(std::sync::atomic::Ordering::SeqCst), + 1 + ); + } } diff --git a/dstack/kms/src/main_service/amd_attest.rs b/dstack/kms/src/main_service/amd_attest.rs index a9a943131..569f964c5 100644 --- a/dstack/kms/src/main_service/amd_attest.rs +++ b/dstack/kms/src/main_service/amd_attest.rs @@ -592,6 +592,7 @@ mod tests { pcr0: Vec::new(), pcr1: Vec::new(), pcr2: Vec::new(), + pcr4: Vec::new(), }, user_data: Vec::new(), timestamp: 0, @@ -786,7 +787,13 @@ mod tests { fn rejects_unsafe_machine_config() { let mut input = valid_input(); input.guest_features = 0; - assert_rejects(input, "guest_features must be non-zero"); + assert_rejects(input, "guest_features must set SNPActive"); + + // DebugSwap hands guest debug state to the host; dstack never launches + // with it, so the KMS must not recompute a matching digest for it. + let mut input = valid_input(); + input.guest_features = 1 | (1 << 5); + assert_rejects(input, "unsupported SEV_FEATURES bits"); let mut input = valid_input(); input.ovmf_sections[0].size = 0; diff --git a/dstack/kms/src/main_service/upgrade_authority.rs b/dstack/kms/src/main_service/upgrade_authority.rs index 8b1c20742..c61ecfe04 100644 --- a/dstack/kms/src/main_service/upgrade_authority.rs +++ b/dstack/kms/src/main_service/upgrade_authority.rs @@ -12,6 +12,7 @@ use http_client::prpc::PrpcClient; use ra_tls::attestation::{AttestationVerifier, VerifiedAttestation, VersionedAttestation}; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; +use tracing::warn; /// The KMS `bootAuth` payload. This is the verifier's `PolicyBootInfo` — the one /// canonical struct shared by the producer (KMS) and the policy input the @@ -49,6 +50,19 @@ pub(crate) fn ensure_app_id_len(app_id: &[u8]) -> Result<()> { Ok(()) } +/// Policy backends left-pad short hex values, so a short field would alias a +/// whitelisted full-width one. +fn ensure_identity_widths(boot_info: &BootInfo) -> Result<()> { + ensure_app_id_len(&boot_info.app_id)?; + if boot_info.compose_hash.len() != 32 { + bail!("compose_hash must be 32 bytes"); + } + if !matches!(boot_info.instance_id.len(), 0 | 20) { + bail!("instance_id must be 20 bytes or empty"); + } + Ok(()) +} + pub(crate) async fn local_kms_boot_info(verifier: &AttestationVerifier) -> Result { let response = app_attest(pad64([0u8; 32])) .await @@ -83,7 +97,7 @@ pub(crate) struct AuthApiInfoResponse { pub app_implementation: String, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct GetInfoResponse { pub is_dev: bool, @@ -102,12 +116,30 @@ async fn http_post(url: &str, body: &impl Serialize) -> Res send_request(reqwest::Client::new().post(url).json(body), url).await } +/// Longest stretch of an auth-backend response quoted back in an error. +/// +/// #525's convention: include the body so a misconfigured backend is +/// diagnosable, bounded so an HTML error page cannot flood the logs. +const MAX_QUOTED_BODY_BYTES: usize = 512; + +/// Take at most [`MAX_QUOTED_BODY_BYTES`] of `body`, ending on a character boundary. +/// +/// Slicing a `String` at a fixed byte index panics when that index lands inside a +/// multi-byte character, and this body is whatever the auth backend sent -- a proxy's +/// error page, or a `reason` string an app owner controls through their `DstackApp` +/// contract. Release binaries build with `panic = "abort"`, so that is the KMS process +/// dying on the boot-authorization path rather than one call failing. Rounding the +/// bound down to a boundary keeps it a byte bound, not a character count. +fn quoted_body(body: &str) -> &str { + &body[..body.floor_char_boundary(MAX_QUOTED_BODY_BYTES)] +} + async fn send_request(req: reqwest::RequestBuilder, url: &str) -> Result { static USER_AGENT: &str = concat!("dstack-kms/", env!("CARGO_PKG_VERSION")); let response = req.header("User-Agent", USER_AGENT).send().await?; let status = response.status(); let body = response.text().await?; - let short_body = &body[..body.len().min(512)]; + let short_body = quoted_body(&body); if !status.is_success() { bail!("auth api {url} returned {status}: {short_body}"); } @@ -118,6 +150,7 @@ async fn send_request(req: reqwest::RequestBuilder, url: &s impl AuthApi { pub async fn is_app_allowed(&self, boot_info: &BootInfo, is_kms: bool) -> Result { + ensure_identity_widths(boot_info)?; match self { AuthApi::Dev { dev } => Ok(BootResponse { is_allowed: true, @@ -222,10 +255,11 @@ pub(crate) async fn ensure_kms_allowed( .context("failed to build KMS boot info from attestation")?; // Workaround: old source KMS instances use the legacy cert format (separate TDX_QUOTE + // EVENT_LOG OIDs) which lacks vm_config, resulting in an empty os_image_hash. - // Fill it from the local KMS's own value. This is safe because mrAggregated already - // validates OS image integrity transitively through the RTMR measurement chain. + // Fill it from the local KMS's own value, so only mrAggregated gates the source image here. + // A source that presents a config but no os_image_hash is not legacy and gets no fallback. // TODO: remove once all source KMS instances use the unified PHALA_RATLS_ATTESTATION format. - if boot_info.os_image_hash.is_empty() { + if boot_info.os_image_hash.is_empty() && attestation.config.is_empty() { + warn!("source KMS uses the legacy cert format, falling back to the local os_image_hash"); let local_info = local_kms_boot_info(verifier) .await .context("failed to get local KMS boot info for os_image_hash fallback")?; @@ -268,7 +302,10 @@ mod tests { } } - fn serve(responses: Vec<&'static str>) -> (String, thread::JoinHandle>) { + fn serve>( + responses: Vec, + ) -> (String, thread::JoinHandle>) { + let responses: Vec = responses.into_iter().map(Into::into).collect(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let handle = thread::spawn(move || { @@ -342,6 +379,43 @@ mod tests { } } + #[rocket::async_test] + async fn short_identities_are_refused_before_they_reach_the_backend() { + let (url, server) = serve(vec![ + r#"{"isAllowed":true,"gatewayAppId":"gateway","reason":"well-formed"}"#, + ]); + let auth = webhook(url); + + let mut aliasing_compose_hash = boot_info(1); + aliasing_compose_hash.compose_hash.truncate(31); + let err = auth + .is_app_allowed(&aliasing_compose_hash, false) + .await + .unwrap_err(); + assert!(err.to_string().contains("compose_hash must be 32 bytes")); + + let mut aliasing_instance_id = boot_info(1); + aliasing_instance_id.instance_id.truncate(19); + let err = auth + .is_app_allowed(&aliasing_instance_id, false) + .await + .unwrap_err(); + assert!(err + .to_string() + .contains("instance_id must be 20 bytes or empty")); + + // `--no-instance-id` deployments carry no instance id. + let mut no_instance_id = boot_info(1); + no_instance_id.instance_id.clear(); + assert!( + auth.is_app_allowed(&no_instance_id, false) + .await + .unwrap() + .is_allowed + ); + assert_eq!(server.join().unwrap().len(), 1); + } + #[rocket::async_test] async fn repeated_authorization_is_never_served_from_a_decision_cache() { let (url, server) = serve(vec![ @@ -404,4 +478,38 @@ mod tests { assert_eq!(recovered.reason, "recovered"); assert_eq!(server.join().unwrap().len(), 2); } + + /// The auth backend's body is quoted into the error that reports a bad + /// response. Byte-slicing it at a fixed offset aborts the KMS when that + /// offset lands inside a multi-byte character, and this runs on every + /// response, including a successful one. + #[rocket::async_test] + async fn a_response_with_a_multibyte_character_on_the_quoting_bound_does_not_abort() { + let mut body = "A".repeat(MAX_QUOTED_BODY_BYTES - 1); + body.push('\u{e9}'); // occupies bytes 511..513, so 512 is mid-character + let (url, server) = serve(vec![body]); + + let error = webhook(url) + .is_app_allowed(&boot_info(40), false) + .await + .unwrap_err(); + + assert!(format!("{error:#}").contains("failed to decode response")); + server.join().unwrap(); + } + + #[test] + fn a_quoted_body_is_bounded_in_bytes_and_never_splits_a_character() { + assert_eq!(quoted_body("short"), "short"); + assert_eq!(quoted_body(&"A".repeat(600)).len(), MAX_QUOTED_BODY_BYTES); + // Every alignment of a 2-, 3- and 4-byte character across the bound. + for pad in MAX_QUOTED_BODY_BYTES - 4..MAX_QUOTED_BODY_BYTES { + for wide in ['\u{e9}', '\u{4e2d}', '\u{1f600}'] { + let body = format!("{}{wide}{}", "A".repeat(pad), "B".repeat(16)); + let quoted = quoted_body(&body); + assert!(quoted.len() <= MAX_QUOTED_BODY_BYTES); + assert!(body.starts_with(quoted)); + } + } + } } diff --git a/dstack/kms/src/onboard_service.rs b/dstack/kms/src/onboard_service.rs index 2d2d8d0af..3bcc42f10 100644 --- a/dstack/kms/src/onboard_service.rs +++ b/dstack/kms/src/onboard_service.rs @@ -112,7 +112,7 @@ impl OnboardRpc for OnboardHandler { validate_onboarding_domain(&request.domain)?; let _bootstrap_guard = self.state.bootstrap_lock.lock().await; let cfg = &self.state.config; - if cfg.bootstrap_info().exists() || cfg.root_ca_key().exists() || cfg.k256_key().exists() { + if cfg.root_keys_exist() { bail!("KMS has already been bootstrapped"); } ensure_self_kms_allowed(cfg, &self.state.attestation_verifier) @@ -141,7 +141,7 @@ impl OnboardRpc for OnboardHandler { validate_onboarding_domain(&request.domain)?; let _bootstrap_guard = self.state.bootstrap_lock.lock().await; let cfg = &self.state.config; - if cfg.root_ca_key().exists() || cfg.k256_key().exists() { + if cfg.root_keys_exist() { bail!("KMS has already been onboarded"); } let source_url = request.source_url.trim_end_matches('/').to_string(); @@ -430,6 +430,70 @@ mod tests { .into_bytes() } + fn auto_bootstrap_config(cert_dir: &std::path::Path) -> KmsConfig { + use rocket::figment::{ + providers::{Format, Toml}, + Figment, + }; + let overrides = format!( + r#" + [core] + cert_dir = "{}" + attest_rpc_cert = false + enforce_self_authorization = false + + [core.onboard] + auto_bootstrap_domain = "kms.example.com" + "#, + cert_dir.display() + ); + Figment::from(rocket::Config::default()) + .merge(Toml::string(crate::config::DEFAULT_CONFIG)) + .merge(Toml::string(&overrides)) + .focus("core") + .extract() + .unwrap() + } + + #[rocket::async_test] + async fn auto_bootstrap_refuses_to_replace_existing_root_keys() { + let cert_dir = tempfile::tempdir().unwrap(); + let cfg = auto_bootstrap_config(cert_dir.path()); + let verifier = AttestationVerifier::load(&cfg.attestation).unwrap(); + + bootstrap_keys(&cfg, &verifier).await.unwrap(); + let root_ca_key = fs::read(cfg.root_ca_key()).unwrap(); + let k256_key = fs::read(cfg.k256_key()).unwrap(); + + // A single missing certificate is enough to make `keys_exists()` false, + // which is what sends the KMS back into onboarding on the next start. + fs::remove_file(cfg.rpc_cert()).unwrap(); + assert!(!cfg.keys_exists()); + + let err = bootstrap_keys(&cfg, &verifier).await.unwrap_err(); + assert!( + format!("{err:#}").contains("already been bootstrapped"), + "unexpected error: {err:#}" + ); + assert_eq!(fs::read(cfg.root_ca_key()).unwrap(), root_ca_key); + assert_eq!(fs::read(cfg.k256_key()).unwrap(), k256_key); + } + + #[rocket::async_test] + async fn auto_bootstrap_overwrites_an_interrupted_bootstrap() { + let cert_dir = tempfile::tempdir().unwrap(); + let cfg = auto_bootstrap_config(cert_dir.path()); + let verifier = AttestationVerifier::load(&cfg.attestation).unwrap(); + + // A crash between the two root key writes. + fs::write(cfg.root_ca_key(), "stale").unwrap(); + assert!(!cfg.root_keys_exist()); + + bootstrap_keys(&cfg, &verifier).await.unwrap(); + assert!(cfg.keys_exists()); + assert_ne!(fs::read(cfg.root_ca_key()).unwrap(), b"stale"); + } + #[test] fn ca_certificate_is_renewed_only_within_the_renewal_window() { let now = UNIX_EPOCH + Duration::from_secs(2_000_000_000); @@ -613,17 +677,18 @@ impl Keys { .await } + /// The root keys go last: see [`KmsConfig::root_keys_exist`]. fn store(&self, cfg: &KmsConfig) -> Result<()> { - self.store_keys(cfg)?; self.store_certs(cfg)?; safe_write(cfg.rpc_domain(), self.rpc_domain.as_bytes())?; + self.store_keys(cfg)?; Ok(()) } fn store_keys(&self, cfg: &KmsConfig) -> Result<()> { safe_write_with_mode(cfg.tmp_ca_key(), self.tmp_ca_key.serialize_pem(), 0o600)?; - safe_write_with_mode(cfg.root_ca_key(), self.ca_key.serialize_pem(), 0o600)?; safe_write_with_mode(cfg.rpc_key(), self.rpc_key.serialize_pem(), 0o600)?; + safe_write_with_mode(cfg.root_ca_key(), self.ca_key.serialize_pem(), 0o600)?; safe_write_with_mode(cfg.k256_key(), self.k256_key.to_bytes(), 0o600)?; Ok(()) } @@ -717,6 +782,14 @@ fn ca_cert_expires_within(cert_pem: &[u8], now: SystemTime, window: Duration) -> pub(crate) async fn bootstrap_keys(cfg: &KmsConfig, verifier: &AttestationVerifier) -> Result<()> { validate_onboarding_domain(&cfg.onboard.auto_bootstrap_domain)?; + // `keys_exists()` wants every key *and* certificate, so losing one derived + // certificate - a restored disk image - sends the KMS back here. Generating + // new root keys then would silently change every app key and orphan every + // encrypted disk, so refuse and let the operator recover the missing file + // instead. Same guard as the `Onboard.Bootstrap` RPC. + if cfg.root_keys_exist() { + bail!("KMS has already been bootstrapped"); + } ensure_self_kms_allowed(cfg, verifier) .await .context("KMS is not allowed to auto-bootstrap")?; diff --git a/dstack/local-key-provider/README.md b/dstack/local-key-provider/README.md index cf3cc09ae..656a437e0 100644 --- a/dstack/local-key-provider/README.md +++ b/dstack/local-key-provider/README.md @@ -7,13 +7,20 @@ dstack repository. For each request it: -1. verifies the guest's TDX quote with DCAP; +1. verifies the guest's TDX quote with DCAP and refuses a TD that is in debug + mode or was launched by a non-production TDX module; 2. checks that the SGX and TDX quotes carry the same quoting-enclave ID; 3. derives a 32-byte key as `SHA-256(SGX sealing key || MRTD || RTMR0..3)`; 4. encrypts the key using the libsodium sealed-box format and the X25519 public key in the TDX report data; and 5. returns the ciphertext with an SGX quote binding its SHA-256 digest. +The derivation covers MRTD and the RTMRs but not `TD_ATTRIBUTES`, and it stays +that way for compatibility with keys already in use. A debug TD running the +same image would therefore derive the production TD's key. dcap-qvl rejects +debug TDs by default; step 1 also checks the TD attributes and TDX module +itself, so the key does not depend on that default. + The wire protocol remains a four-byte big-endian JSON length followed by a `{"quote":[...]}` request. The response contains `encrypted_key` and `provider_quote` byte arrays. @@ -24,6 +31,6 @@ provider under Gramine. The container build uses the repository workspace's lockfile. Tagged releases (`local-key-provider-v*`) publish the image as -`docker.io/dstacktee/local-key-provider:`. The corresponding GitHub +`ghcr.io/dstack-tee/local-key-provider:`. The corresponding GitHub release records the image digest and enclave measurements, attaches the full SIGSTRUCT metadata, and links to its Sigstore build-provenance attestation. diff --git a/dstack/local-key-provider/src/error.rs b/dstack/local-key-provider/src/error.rs index 8c3492300..ff974eeab 100644 --- a/dstack/local-key-provider/src/error.rs +++ b/dstack/local-key-provider/src/error.rs @@ -26,6 +26,9 @@ pub enum ProviderError { #[error("SGX and TDX quoting-enclave identifiers do not match")] QeIdMismatch, + #[error("untrusted TD: {0}")] + UntrustedTd(&'static str), + #[error("invalid public key in TDX report data")] InvalidPublicKey, diff --git a/dstack/local-key-provider/src/provider.rs b/dstack/local-key-provider/src/provider.rs index 154200aea..9445648e6 100644 --- a/dstack/local-key-provider/src/provider.rs +++ b/dstack/local-key-provider/src/provider.rs @@ -4,7 +4,7 @@ use dcap_qvl::{ collateral::CollateralClient, - quote::{Quote, TDReport10}, + quote::{Quote, TDAttributes, TDReport10}, }; use sha2::{Digest, Sha256}; use tracing::{debug, info}; @@ -66,6 +66,7 @@ impl KeyProvider { let tdx_report = report.report.as_td10().copied().ok_or_else(|| { ProviderError::QuoteVerification("verified quote is not a TDX quote".into()) })?; + require_production_td(&tdx_report)?; debug!( tcb_status = %report.status, advisories = ?report.advisory_ids, @@ -75,6 +76,40 @@ impl KeyProvider { } } +/// Refuse a TD that the host launched in a state that does not protect it. +/// +/// `fetch_and_verify` already rejects these TDs under dcap-qvl's default +/// policy. The check is repeated here, following dcap-qvl's `validate_td10`, +/// so that the sealing key does not depend on that default: the key +/// derivation does not cover `td_attributes`, so a debug TD running the same +/// image would derive the production TD's key. The `mr_signer_seam` check +/// matches the guest's `validate_tcb`. +fn require_production_td(report: &TDReport10) -> Result<(), ProviderError> { + let attributes = TDAttributes::parse(report.td_attributes) + .map_err(|error| ProviderError::QuoteVerification(error.to_string()))?; + if attributes.tud & 0x01 != 0 { + return Err(ProviderError::UntrustedTd("debug mode is enabled")); + } + if attributes.tud != 0 + || attributes.sec.reserved_lower != 0 + || attributes.sec.reserved_bit29 + || attributes.other.reserved != 0 + { + return Err(ProviderError::UntrustedTd( + "reserved bits in TD attributes are set", + )); + } + if !attributes.sec.sept_ve_disable { + return Err(ProviderError::UntrustedTd("SEPT_VE_DISABLE is not enabled")); + } + if report.mr_signer_seam != [0_u8; 48] { + return Err(ProviderError::UntrustedTd( + "TD was launched by a non-production TDX module", + )); + } + Ok(()) +} + fn parse_quote(kind: &'static str, raw_quote: &[u8]) -> Result { Quote::parse(raw_quote).map_err(|error| ProviderError::QuoteParse { kind, @@ -114,9 +149,75 @@ fn measurements(report: &TDReport10) -> Vec { mod tests { use super::*; + const TDX_QUOTE: &[u8] = include_bytes!("../../ra-tls/assets/tdx_quote"); + /// `td_attributes` starts 120 bytes into the TD report body, which itself + /// starts 48 bytes into the quote. + const TD_ATTRIBUTES_OFFSET: usize = 48 + 120; + + fn report(quote: &[u8]) -> TDReport10 { + Quote::parse(quote) + .unwrap() + .report + .as_td10() + .copied() + .unwrap() + } + + /// The recorded quote with `td_attributes[byte]` XOR'd with `mask`. + fn quote_with_attributes(byte: usize, mask: u8) -> Vec { + let mut quote = TDX_QUOTE.to_vec(); + quote[TD_ATTRIBUTES_OFFSET + byte] ^= mask; + quote + } + + fn rejection(quote: &[u8]) -> Option<&'static str> { + match require_production_td(&report(quote)) { + Err(ProviderError::UntrustedTd(reason)) => Some(reason), + _ => None, + } + } + + #[test] + fn td_attributes_offset_is_the_recorded_one() { + assert_eq!( + report(TDX_QUOTE).td_attributes, + TDX_QUOTE[TD_ATTRIBUTES_OFFSET..TD_ATTRIBUTES_OFFSET + 8] + ); + } + + #[test] + fn accepts_the_recorded_production_td() { + assert!(require_production_td(&report(TDX_QUOTE)).is_ok()); + } + + #[test] + fn refuses_untrusted_td_attributes() { + let cases = [ + (0, 0x01, "debug mode is enabled"), + (0, 0x02, "reserved bits in TD attributes are set"), + (1, 0x01, "reserved bits in TD attributes are set"), + (3, 0x20, "reserved bits in TD attributes are set"), + (4, 0x01, "reserved bits in TD attributes are set"), + (3, 0x10, "SEPT_VE_DISABLE is not enabled"), + ]; + for (byte, mask, reason) in cases { + assert_eq!(rejection("e_with_attributes(byte, mask)), Some(reason)); + } + } + + #[test] + fn refuses_a_td_from_a_non_production_tdx_module() { + let mut report = report(TDX_QUOTE); + report.mr_signer_seam[0] = 0x01; + assert!(matches!( + require_production_td(&report), + Err(ProviderError::UntrustedTd(_)) + )); + } + #[test] fn extracts_all_key_derivation_measurements_in_wire_order() { - let quote = Quote::parse(include_bytes!("../../ra-tls/assets/tdx_quote")).unwrap(); + let quote = Quote::parse(TDX_QUOTE).unwrap(); let report = quote.report.as_td10().unwrap(); let output = measurements(report); diff --git a/dstack/ra-tls/src/cert.rs b/dstack/ra-tls/src/cert.rs index ed3d71583..09c46b02f 100644 --- a/dstack/ra-tls/src/cert.rs +++ b/dstack/ra-tls/src/cert.rs @@ -113,8 +113,16 @@ impl CaCert { .maybe_app_id(app_id) .maybe_app_info(app_info.as_ref()) .special_usage(usage) - .maybe_not_before(cfg.not_before.map(unix_time_to_system_time)) - .maybe_not_after(cfg.not_after.map(unix_time_to_system_time)) + .maybe_not_before( + cfg.not_before + .map(|secs| unix_time_to_system_time(secs, "not_before")) + .transpose()?, + ) + .maybe_not_after( + cfg.not_after + .map(|secs| unix_time_to_system_time(secs, "not_after")) + .transpose()?, + ) .build(); self.sign(req).context("Failed to sign certificate") } @@ -427,8 +435,16 @@ fn add_ext(params: &mut CertificateParams, oid: &[u64], content: impl AsRef<[u8] .push(CustomExtension::from_oid_content(oid, content)); } -fn unix_time_to_system_time(secs: u64) -> SystemTime { - UNIX_EPOCH + Duration::from_secs(secs) +/// The last timestamp representable by an RFC 5280 certificate. +pub const MAX_CERT_VALIDITY_SECS: u64 = 253_402_300_799; + +fn unix_time_to_system_time(secs: u64, field: &str) -> Result { + if secs > MAX_CERT_VALIDITY_SECS { + bail!("{field} {secs} is past the last representable certificate time"); + } + UNIX_EPOCH + .checked_add(Duration::from_secs(secs)) + .with_context(|| format!("{field} {secs} overflows the system clock range")) } impl CertRequest<'_, KeyPair> { @@ -697,6 +713,59 @@ mod tests { assert_eq!(decompressed, large_data); } + #[test] + fn validates_certificate_timestamps() { + assert!(unix_time_to_system_time(MAX_CERT_VALIDITY_SECS, "not_after").is_ok()); + assert!(unix_time_to_system_time(MAX_CERT_VALIDITY_SECS + 1, "not_after").is_err()); + assert!(unix_time_to_system_time(u64::MAX, "not_after").is_err()); + } + + #[test] + fn signing_a_csr_with_an_unrepresentable_not_after_fails_without_aborting() { + let ca_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap(); + let mut ca_params = CertificateParams::new(vec![]).unwrap(); + ca_params.is_ca = IsCa::Ca(BasicConstraints::Constrained(1)); + let ca_cert = ca_params.self_signed(&ca_key).unwrap(); + let ca = CaCert::from_parts(KeyPair::from_pem(&ca_key.serialize_pem()).unwrap(), ca_cert); + + let key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap(); + let csr = CertSigningRequestV2 { + confirm: "please sign cert:".to_string(), + pubkey: key.public_key_der(), + config: CertConfigV2 { + org_name: None, + subject: "test.example.com".to_string(), + subject_alt_names: vec![], + usage_server_auth: true, + usage_client_auth: false, + ext_quote: false, + ext_app_info: false, + not_before: None, + not_after: Some(u64::MAX), + }, + attestation: Attestation { + quote: AttestationQuote::DstackTdx(TdxQuote { + quote: vec![], + event_log: vec![], + }), + runtime_events: vec![], + report_data: [0u8; 64], + config: "".into(), + report: (), + } + .into_versioned(), + }; + + let error = match ca.sign_csr(&csr, None, "app:custom") { + Ok(_) => panic!("an unrepresentable not_after must be rejected"), + Err(error) => error, + }; + assert!( + format!("{error:#}").contains("not_after"), + "the error must name the field: {error:#}" + ); + } + #[test] fn test_csr_v2_scale_encoding_stable() { let csr = CertSigningRequestV2 { diff --git a/dstack/scripts/install.sh b/dstack/scripts/install.sh index 736c18745..1540f2346 100755 --- a/dstack/scripts/install.sh +++ b/dstack/scripts/install.sh @@ -163,14 +163,16 @@ script_checkout() { return 1 } +# Sets `checkout`. Runs in the calling shell rather than a command +# substitution so the temporary checkout it may create stays visible to the +# exit trap that removes it. resolve_source() { if is_checkout "."; then - abs_dir "." + checkout=$(abs_dir ".") return 0 fi if checkout=$(script_checkout); then - echo "$checkout" return 0 fi @@ -211,7 +213,7 @@ resolve_source() { ) >&2 fi - abs_dir "$src" + checkout=$(abs_dir "$src") } validate_prefix() { @@ -246,7 +248,7 @@ fi need_cmd cargo need_cmd install -checkout=$(resolve_source) +resolve_source core_checkout=$(core_dir "$checkout") bin_dir="$prefix/bin" diff --git a/dstack/sev-snp-qvl/src/lib.rs b/dstack/sev-snp-qvl/src/lib.rs index 1fa4da6f5..68883363b 100644 --- a/dstack/sev-snp-qvl/src/lib.rs +++ b/dstack/sev-snp-qvl/src/lib.rs @@ -27,6 +27,9 @@ const VLEK_CERT_GUID: [u8; 16] = [ 0xa8, 0x07, 0x4b, 0xc2, 0xa2, 0x5a, 0x48, 0x3e, 0xaa, 0xe6, 0x39, 0xc0, 0x45, 0xa0, 0xb8, 0xa1, ]; const CERT_TABLE_ENTRY_SIZE: usize = 24; +/// Size of an AMD SEV-SNP attestation report, for every version the `sev` +/// crate decodes. +const AMD_SNP_REPORT_LEN: usize = 1184; pub const AMD_KDS_DEFAULT_BASE_URL: &str = "https://kdsintf.amd.com/vcek/v1"; const AMD_KDS_CONNECT_TIMEOUT: Duration = Duration::from_secs(30); const AMD_KDS_REQUEST_TIMEOUT: Duration = Duration::from_secs(120); @@ -142,8 +145,7 @@ impl QuoteVerifier { if !cert_chain.is_empty() { return self.verify(report, cert_chain, expected_report_data); } - let report_obj = AttestationReport::from_bytes(report) - .map_err(|err| anyhow!("failed to parse amd sev-snp report: {err}"))?; + let report_obj = decode_amd_snp_report(report)?; let mut errors = Vec::new(); for product in amd_snp_product_candidates_for_report(&report_obj)? { match kds_client @@ -421,15 +423,34 @@ pub fn verify_amd_snp_attestation( ) } -pub fn parse_amd_snp_report(report_bytes: &[u8]) -> Result { - if report_bytes.len() != 1184 { +/// Decode an attestation report, rejecting anything that is not report-sized +/// first. +/// +/// `AttestationReport::from_bytes` indexes `bytes[0..4]` and, for a version it +/// does not know, `bytes[392]`, without looking at the length: a report shorter +/// than that aborts the process, because release builds are `panic = "abort"` +/// (`dstack/Cargo.toml`). The report arrives in an unverified attestation blob, +/// so the length has to be checked on every path that reaches this parser, not +/// on the three that happened to. +fn decode_amd_snp_report(report_bytes: &[u8]) -> Result { + if report_bytes.len() != AMD_SNP_REPORT_LEN { bail!( - "invalid amd sev-snp report length: expected 1184 bytes, got {}", + "invalid amd sev-snp report length: expected {AMD_SNP_REPORT_LEN} bytes, got {}", report_bytes.len() ); } - let report = AttestationReport::from_bytes(report_bytes) - .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; + AttestationReport::from_bytes(report_bytes) + .map_err(|err| anyhow!("failed to parse amd sev-snp report: {err}")) +} + +/// Decode an attestation report's fields without checking its authenticity. +/// +/// This does not verify the report signature or the AMD certificate chain. +/// Use [`verify_amd_snp_attestation`] when authenticity is in question; use +/// this only where the signature is established elsewhere, or for the guest's +/// own report. +pub fn parse_unverified_amd_snp_report(report_bytes: &[u8]) -> Result { + let report = decode_amd_snp_report(report_bytes)?; parsed_amd_snp_report_from_report(&report) } @@ -491,14 +512,7 @@ fn verify_amd_snp_attestation_with_certs_and_arks( vcek_bytes: CertBytes, ark_for_product: impl Fn(AmdSnpProduct) -> (CertBytes, bool), ) -> Result { - if report_bytes.len() != 1184 { - bail!( - "invalid amd sev-snp report length: expected 1184 bytes, got {}", - report_bytes.len() - ); - } - let report = AttestationReport::from_bytes(report_bytes) - .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; + let report = decode_amd_snp_report(report_bytes)?; let mut errors = Vec::new(); for product in amd_snp_product_candidates_for_report(&report)? { let (ark, external_root) = ark_for_product(product); @@ -526,14 +540,7 @@ fn verify_amd_snp_attestation_with_cert_chain( vcek_bytes: CertBytes, external_root: bool, ) -> Result { - if report_bytes.len() != 1184 { - bail!( - "invalid amd sev-snp report length: expected 1184 bytes, got {}", - report_bytes.len() - ); - } - let report = AttestationReport::from_bytes(report_bytes) - .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; + let report = decode_amd_snp_report(report_bytes)?; let ark = parse_certificate(&ark_bytes, "ark")?; let ask = parse_certificate(&ask_bytes, "ask")?; @@ -988,6 +995,36 @@ mod tests { assert_eq!(stale_vcek_reported.tcb_status(), "OutOfDate"); } + /// `fetch_and_verify` takes the KDS branch whenever `cert_chain` is empty, + /// and that branch reached `AttestationReport::from_bytes` with whatever + /// the requester sent. `from_bytes` reads `bytes[0..4]` and, for an unknown + /// version, `bytes[392]`, unguarded — so an empty report is an abort, not a + /// rejected request. Both shapes come straight out of a `POST /verify` + /// body. + #[test] + fn a_short_report_without_a_cert_chain_is_rejected_rather_than_parsed() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to build a test runtime"); + let verifier = QuoteVerifier::new_prod(); + let kds = AmdKdsClient::new().expect("failed to build a KDS client"); + + // Empty: `&bytes[0..4]` is out of range. + // Four bytes of version 3: `bytes[392]` is out of range. + for report in [vec![], vec![0x03, 0x00, 0x00, 0x00], vec![0u8; 1183]] { + let err = runtime + .block_on(verifier.fetch_and_verify(&kds, &report, &[], &[0u8; 64])) + .expect_err("a short report must be rejected"); + assert!( + err.to_string() + .contains("invalid amd sev-snp report length"), + "unexpected error for a {}-byte report: {err:#}", + report.len() + ); + } + } + #[test] fn missing_cert_chain_fails_closed() { let report = vec![0u8; 1184]; diff --git a/dstack/tpm-qvl/src/verify.rs b/dstack/tpm-qvl/src/verify.rs index 21c31167a..04bda0329 100644 --- a/dstack/tpm-qvl/src/verify.rs +++ b/dstack/tpm-qvl/src/verify.rs @@ -115,6 +115,39 @@ pub fn verify_quote_with_ca( }); } + // The attested bank fixes both the digest length of every quoted PCR and + // the name `PcrValue` must carry, and neither was checked. Length matters + // because compute_pcr_digest() concatenates the values with nothing marking + // where one ends: a caller can re-split the preimage of a genuine, + // AK-signed pcr_digest into differently sized values, match the digest, and + // have VerifiedReport::get_pcr() hand out a PCR the TPM never held. The + // name matters because it is the only thing a consumer of VerifiedReport + // can read to learn which bank it is looking at. + const SHA256_BANK: &str = "sha256"; + const SHA256_DIGEST_LEN: usize = 32; + for pcr in "e.pcr_values { + if pcr.algorithm != SHA256_BANK { + return Err(VerificationError { + status: status.clone(), + error: anyhow!( + "PCR {} is labelled {:?}, but the quote attests the {SHA256_BANK} bank", + pcr.index, + pcr.algorithm + ), + }); + } + if pcr.value.len() != SHA256_DIGEST_LEN { + return Err(VerificationError { + status: status.clone(), + error: anyhow!( + "PCR {} value is {} bytes, expected {SHA256_DIGEST_LEN} for the {SHA256_BANK} bank", + pcr.index, + pcr.value.len() + ), + }); + } + } + let computed_pcr_digest = compute_pcr_digest("e.pcr_values).map_err(|e| VerificationError { status: status.clone(), @@ -687,3 +720,127 @@ fn verify_ak_chain_with_collateral( } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{QuoteCollateral, GCP_ROOT_CA}; + + /// Everything a quote must satisfy to reach `compute_pcr_digest` is + /// reproducible without a TPM: the PCR selection list lives in the + /// still-unsigned TPMS_ATTEST, and `pcr_digest` is a plain SHA-256 over the + /// supplied PCR values. Both gates run before the AK signature is checked. + fn attest_message(pcr_indices: &[u8], hash_alg: u16, pcr_digest: &[u8]) -> Vec { + let mut msg = Vec::new(); + msg.extend_from_slice(&0xff544347u32.to_be_bytes()); // magic + msg.extend_from_slice(&0x8018u16.to_be_bytes()); // TPM_ST_ATTEST_QUOTE + msg.extend_from_slice(&0u16.to_be_bytes()); // qualified_signer + msg.extend_from_slice(&0u16.to_be_bytes()); // qualified_data + msg.extend_from_slice(&0u64.to_be_bytes()); // clock + msg.extend_from_slice(&0u32.to_be_bytes()); // reset_count + msg.extend_from_slice(&0u32.to_be_bytes()); // restart_count + msg.push(1); // safe + msg.extend_from_slice(&0u64.to_be_bytes()); // firmware_version + msg.extend_from_slice(&1u32.to_be_bytes()); // one selection + msg.extend_from_slice(&hash_alg.to_be_bytes()); + msg.push(2); // sizeof_select + let mut bitmap = [0u8; 2]; + for index in pcr_indices { + bitmap[(index / 8) as usize] |= 1 << (index % 8); + } + msg.extend_from_slice(&bitmap); + msg.extend_from_slice(&(pcr_digest.len() as u16).to_be_bytes()); + msg.extend_from_slice(pcr_digest); + msg + } + + fn quote_of(pcr_values: Vec, message: Vec) -> TpmQuote { + TpmQuote { + message, + signature: Vec::new(), + pcr_values, + ak_cert: Vec::new(), + platform: Platform::Gcp, + event_log: Vec::new(), + } + } + + fn empty_collateral() -> QuoteCollateral { + QuoteCollateral { + cert_chain_pem: String::new(), + crls: Vec::new(), + root_ca_crl: None, + } + } + + /// `compute_pcr_digest` concatenates the PCR values with nothing marking + /// where one ends and the next begins, so a caller can move the boundary. + /// The signed `pcr_digest` still matches, the indices still match the signed + /// selection, and `VerifiedReport::get_pcr` then hands out a PCR 4 value + /// that no TPM ever held. + #[test] + fn rejects_pcr_values_whose_lengths_do_not_match_the_attested_bank() { + let pcr4 = vec![0xaa; 32]; + let pcr7 = vec![0xbb; 32]; + let mut concatenated = pcr4.clone(); + concatenated.extend_from_slice(&pcr7); + let genuine_digest = Sha256::digest(&concatenated).to_vec(); + + // Same bytes, boundary moved one byte left. + let shifted = vec![ + PcrValue { + index: 4, + algorithm: "sha256".into(), + value: concatenated[..31].to_vec(), + }, + PcrValue { + index: 7, + algorithm: "sha256".into(), + value: concatenated[31..].to_vec(), + }, + ]; + let message = attest_message(&[4, 7], 0x000b, &genuine_digest); + let err = match verify_quote_with_ca( + "e_of(shifted, message), + &empty_collateral(), + GCP_ROOT_CA, + ) { + Ok(report) => panic!( + "a re-split PCR list verified; get_pcr(4) = {}", + hex::encode(report.get_pcr(4).expect("PCR 4")) + ), + Err(err) => err.error.to_string(), + }; + assert!( + err.contains("PCR 4") && err.contains("31 bytes"), + "unexpected error: {err}" + ); + } + + /// `PcrValue::algorithm` is the only place a consumer of `VerifiedReport` + /// can read which bank the values came from, and nothing compared it against + /// the bank the signed TPMS_ATTEST names. + #[test] + fn rejects_pcr_values_that_misname_the_attested_bank() { + let pcr4 = vec![0xaa; 32]; + let digest = Sha256::digest(&pcr4).to_vec(); + let values = vec![PcrValue { + index: 4, + algorithm: "sha384".into(), + value: pcr4, + }]; + let message = attest_message(&[4], 0x000b, &digest); + let err = match verify_quote_with_ca( + "e_of(values, message), + &empty_collateral(), + GCP_ROOT_CA, + ) { + Ok(_) => panic!("a mislabelled PCR bank verified"), + Err(err) => err.error.to_string(), + }; + assert!( + err.contains("PCR 4") && err.contains("sha384") && err.contains("sha256"), + "unexpected error: {err}" + ); + } +} diff --git a/dstack/verifier/README.md b/dstack/verifier/README.md index 232ede636..8529dc1f6 100644 --- a/dstack/verifier/README.md +++ b/dstack/verifier/README.md @@ -169,7 +169,7 @@ check with the policy fields emitted by `/verify`: accepted OS image, ```yaml services: dstack-verifier: - image: dstacktee/dstack-verifier:latest + image: ghcr.io/dstack-tee/dstack-verifier:latest ports: - "8080:8080" restart: unless-stopped diff --git a/dstack/verifier/builder/README.md b/dstack/verifier/builder/README.md index 9e409e925..728028156 100644 --- a/dstack/verifier/builder/README.md +++ b/dstack/verifier/builder/README.md @@ -59,17 +59,17 @@ GIT_REV=HEAD \ IMAGE_VERSION=0.6.0 \ IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack \ OCI_TAR=/tmp/verifier.oci.tar \ - ./build-image.sh dstacktee/dstack-verifier:0.6.0 + ./build-image.sh ghcr.io/dstack-tee/dstack-verifier:0.6.0 python3 -c 'import json,tarfile;t=tarfile.open("/tmp/verifier.oci.tar");print(json.load(t.extractfile("index.json"))["manifests"][0]["digest"])' ``` `IMAGE_VERSION` is part of the image metadata, so it must match the release for the digests to match. The printed digest is what the registry reports for -`dstacktee/dstack-verifier:0.6.0`; compare it with: +`ghcr.io/dstack-tee/dstack-verifier:0.6.0`; compare it with: ```bash -docker buildx imagetools inspect dstacktee/dstack-verifier:0.6.0 --format '{{.Manifest.Digest}}' +docker buildx imagetools inspect ghcr.io/dstack-tee/dstack-verifier:0.6.0 --format '{{.Manifest.Digest}}' ``` Note that the release also publishes a `:latest` tag pointing at the same diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 191736726..0d8f1add9 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -1706,6 +1706,7 @@ mod tests { pcr0: vec![0x10; 48], pcr1: vec![0x11; 48], pcr2: vec![0x12; 48], + pcr4: vec![0x14; 48], }, user_data: Vec::new(), timestamp: 0, @@ -1726,6 +1727,7 @@ mod tests { pcr0: vec![0x10; 48], pcr1: vec![0x11; 48], pcr2: vec![0x12; 48], + pcr4: vec![0x14; 48], }; let nitro_config: VmConfig = serde_json::from_value(serde_json::json!({ "os_image_hash": hex::encode(nitro_pcrs.image_hash()), @@ -1743,6 +1745,7 @@ mod tests { pcr0: vec![0; 48], pcr1: vec![0; 48], pcr2: vec![0; 48], + pcr4: vec![0; 48], }; assert!(verifier .verify_os_image_hash_for_nitro_enclave(&nitro_config, &debug_nitro) diff --git a/dstack/vmm/Cargo.toml b/dstack/vmm/Cargo.toml index 22387fcef..0850c1697 100644 --- a/dstack/vmm/Cargo.toml +++ b/dstack/vmm/Cargo.toml @@ -68,6 +68,7 @@ tar.workspace = true tempfile.workspace = true wait-timeout.workspace = true listenfd.workspace = true +vfio-ioctls.workspace = true [dev-dependencies] insta.workspace = true diff --git a/dstack/vmm/netd-rpc/proto/netd_rpc.proto b/dstack/vmm/netd-rpc/proto/netd_rpc.proto index 058a83a93..70ec8b991 100644 --- a/dstack/vmm/netd-rpc/proto/netd_rpc.proto +++ b/dstack/vmm/netd-rpc/proto/netd_rpc.proto @@ -26,6 +26,9 @@ message PrepareBridgeRequest { bool filtered = 5; // virtio-net queue pairs. Zero or one creates a single-queue TAP. uint32 queues = 6; + // The VM's working directory on the host. Only logged, so an operator can + // get from a TAP name back to the VM; never used for a decision. + string workdir = 7; } message PrepareMacvtapRequest { @@ -37,6 +40,9 @@ message PrepareMacvtapRequest { string mode = 5; // virtio-net queue pairs. Zero or one creates a single-queue device. uint32 queues = 6; + // The VM's working directory on the host. Only logged; see + // PrepareBridgeRequest.workdir. + string workdir = 7; } // What netd built, so the caller can verify it before handing it to QEMU. @@ -48,6 +54,45 @@ message PreparedInterface { uint32 queues = 3; } +// Names one VM of one VMM instance. +message VmRef { + string instance_id = 1; + string vm_id = 2; +} + +message RemoveVmResponse { + // How many interfaces were deleted. + uint32 removed = 1; +} + +message InterfaceName { + string tap = 1; +} + +message ListInterfacesRequest { + // Only interfaces recorded as this instance's. Empty lists every interface + // netd owns, including ones it cannot attribute. + string instance_id = 1; +} + +// One host resource netd holds. +message InterfaceRecord { + string tap = 1; + // "tap", "macvtap", or "binding" for an nwfilter binding whose interface is + // already gone. + string kind = 2; + // The owner, when the interface carries an ownership record that + // re-derives its name. Absent means the owner is unknown, not that there is + // none. + optional string instance_id = 3; + optional string vm_id = 4; + optional uint32 nic_index = 5; +} + +message InterfaceList { + repeated InterfaceRecord interfaces = 1; +} + message CheckInterfaceRequest { InterfaceIdentity identity = 1; // Also require the nwfilter binding. @@ -64,6 +109,17 @@ service Netd { // Delete an interface and its binding. Deleting one that does not exist // succeeds. rpc RemoveInterface(InterfaceIdentity) returns (google.protobuf.Empty); + // Delete every interface and binding one VM could hold, by deriving each + // name its identity can produce. Reaches interfaces the caller has no record + // of: one a crash left before it was recorded, or one whose NIC the VM no + // longer has. + rpc RemoveVm(VmRef) returns (RemoveVmResponse); + // Delete one interface by name. For interfaces nothing can attribute, such + // as ones built before netd recorded ownership. The name must be one netd + // could have created. + rpc RemoveInterfaceByName(InterfaceName) returns (google.protobuf.Empty); + // Everything netd holds, read off the host. + rpc ListInterfaces(ListInterfacesRequest) returns (InterfaceList); // Verify an interface, and optionally its binding, exists. For operations // and integration diagnostics; launches use the Prepare calls. rpc CheckInterface(CheckInterfaceRequest) returns (google.protobuf.Empty); diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 62bf381be..60eec34b9 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -137,6 +137,12 @@ message VmConfiguration { // Development-only TEE ABI to simulate for this VM. When set, the VMM // launches a non-TEE VM and provides an instance-specific simulator config. optional string simulated_tee = 21; + // Host-side preallocation for the VM data disk: "off", "metadata", + // "falloc" or "full", matching qemu-img's preallocation modes. Unset uses + // the node default (cvm.disk_prealloc). The field itself never enters the + // app identity, but "falloc" and "full" reserve host blocks and are only + // accepted with storage_discard=false in compose_file. + optional string disk_prealloc = 22; } // Per-VM networking configuration. @@ -229,7 +235,7 @@ message UpdateVmRequest { repeated NetworkingConfig networks = 20; } -// Parameters for listing CVMs with pagination and keyword filtering. +// Parameters for listing CVMs with pagination and filtering. message StatusRequest { // List of VM IDs repeated string ids = 1; @@ -241,6 +247,9 @@ message StatusRequest { uint32 page = 4; // Page size uint32 page_size = 5; + // Filter by VM status: running, stopped, exited, stopping, or removing. + // Empty means all statuses. + string status = 6; } // Summary of the current fleet status. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 1675aec25..c63fcff74 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -3,7 +3,10 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - config::{Config, Networking, NetworkingMode, NicNetworking, ProcessAnnotation, Protocol}, + config::{ + Config, DiskPrealloc, Networking, NetworkingMode, NicNetworking, ProcessAnnotation, + Protocol, + }, logrotate, netd::{self, InterfaceIdentity, PrepareBridgeRequest, PrepareMacvtapRequest}, }; @@ -136,6 +139,11 @@ pub struct Manifest { pub networks: Vec, #[serde(default)] pub volumes: Vec, + /// Host-side preallocation used when the data disk is created. Resolved + /// once at deployment from the request and the node default, so a later + /// change to `cvm.disk_prealloc` leaves existing VMs alone. + #[serde(default)] + pub disk_prealloc: DiskPrealloc, } impl Manifest { @@ -300,6 +308,14 @@ pub(crate) enum PullStatus { Failed(String), } +/// First delay before a removal asks netd again to release a VM's interfaces. +#[cfg(not(test))] +const RELEASE_RETRY_INITIAL: Duration = Duration::from_secs(2); +#[cfg(test)] +const RELEASE_RETRY_INITIAL: Duration = Duration::from_millis(20); +/// Longest delay between those attempts. +const RELEASE_RETRY_MAX: Duration = Duration::from_secs(60); + #[derive(Clone)] pub struct App { pub config: Arc, @@ -307,6 +323,9 @@ pub struct App { state: Arc>, /// Pull status for registry images: tag → status. pub(crate) pull_status: Arc>>, + /// One lock per VM, held across a launch or a teardown. See + /// [`App::launch_lock`]. + launch_locks: Arc>>>>, } const GUEST_AGENT_RPC_TIMEOUT: Duration = Duration::from_secs(30); @@ -334,12 +353,42 @@ impl App { state: Arc::new(Mutex::new(AppState { cid_pool, vms: HashMap::new(), + removing: HashSet::new(), })), config: Arc::new(config), pull_status: Arc::new(Mutex::new(std::collections::HashMap::new())), + launch_locks: Arc::new(Mutex::new(HashMap::new())), } } + /// Serializes everything that creates or deletes one VM's host interfaces. + /// + /// A launch reads whether QEMU is already up and then spends many awaits -- + /// a GPU reset, a whole netd conversation, building the QEMU arguments -- + /// before it launches anything. Nothing used to cover that window. Two + /// `StartVm` calls, or one racing the auto-restart timer, could both read + /// "not running", and the loser's sweep would delete the TAPs the winner's + /// QEMU was already holding open: a live VM silently loses its networking, + /// and the loser's error path then clears the winner's record of it. The + /// authoritative rejection lives in the supervisor, which is reached long + /// after the damage is done. + /// + /// A tokio mutex, because it is held across awaits. Per VM, because a slow + /// start must not stall unrelated ones. Taken by stop and by removal as + /// well as by start: those delete the same interfaces from the other side. + pub(crate) async fn launch_lock(&self, id: &str) -> tokio::sync::OwnedMutexGuard<()> { + self.launch_lock_handle(id).lock_owned().await + } + + fn launch_lock_handle(&self, id: &str) -> Arc> { + let mut locks = self.launch_locks.lock().or_panic("mutex poisoned"); + // A VM that is neither starting nor stopping leaves the map holding + // the only reference, so the map stays the size of what is in flight + // rather than of every VM this process has ever touched. + locks.retain(|_, lock| Arc::strong_count(lock) > 1); + locks.entry(id.to_string()).or_default().clone() + } + pub async fn load_vm( &self, work_dir: impl AsRef, @@ -403,6 +452,19 @@ impl App { Ok(()) } + /// Refuses an operation on a VM that is being removed. + /// + /// Cheap, and taken before the launch lock as well as under it. Removal + /// holds that lock until the VM has exited -- hours, by its own estimate -- + /// so anything that only asked afterwards would wait the removal out in + /// order to be told no. + pub(crate) fn refuse_if_removing(&self, id: &str) -> Result<()> { + if self.lock().is_removing(id) { + bail!("VM is being removed"); + } + Ok(()) + } + pub async fn start_vm(&self, id: &str) -> Result<()> { self.start_vm_with_restart_policy(id, true).await } @@ -417,13 +479,24 @@ impl App { vm.state.auto_restart.reset(); } } - { - let state = self.lock(); - if let Some(vm) = state.get(id) { - if vm.state.removing { - bail!("VM is being removed"); - } - } + // Before the lock as well as after it. Removal holds the lock until the + // VM has exited, so a launch that only asked afterwards would wait that + // out -- hours, by removal's own estimate -- to be told no. Asking + // first is not sufficient on its own, because the marker can be set + // while this waits; asking again under the lock is what makes it + // authoritative. + self.refuse_if_removing(id)?; + // Everything below reads whether this VM is running and acts on the + // answer for as long as the launch takes. See [`App::launch_lock`]. + let launch = self.launch_lock(id).await; + self.refuse_if_removing(id)?; + // A restart decided before a stop must not outlive it. The decision + // read `started` from disk; `stop_vm` writes it false under this lock, + // so re-reading it here is what makes the stop stick. An explicit start + // sets the flag itself and has nothing to re-read. + if !reset_restart_policy && !self.work_dir(id)?.started().unwrap_or(false) { + debug!(id, "skipping automatic restart: the VM was stopped"); + return Ok(()); } self.sync_dynamic_config(id)?; let is_running = self @@ -470,31 +543,52 @@ impl App { }) .await .context("GPU sanitization task failed")??; - if let Err(error) = self - .prepare_netd_networks(&vm_config, &mut runtime_networks) - .await + self.prepare_netd_networks(&vm_config, &mut runtime_networks) + .await?; + // Off the async executor: this writes the guest config files and, + // when the VM preallocates, creates the data disk. `full` on a + // large disk writes the whole thing, which is minutes of blocking + // work, and even `falloc` is not instant. + // + // The launch lock travels into the task and back out. A dropped + // request -- a client that gave up, a proxy timeout -- cancels + // this future but not the blocking work, and the disk is only + // published when that work finishes. Leaving the lock with the + // future would let the next attempt run qemu-img against the same + // paths while the abandoned one is still writing them. + let qemu_config = vm_config.clone(); + let app_config = self.config.clone(); + let qemu_workdir = work_dir.path().to_path_buf(); + let qemu_devices = devices.clone(); + let qemu_networks = runtime_networks.clone(); + let (launch, configured) = match tokio::task::spawn_blocking(move || { + let result = qemu_config.config_qemu( + &qemu_workdir, + &app_config.cvm, + &qemu_devices, + &qemu_networks, + ); + (launch, result) + }) + .await { - let _ = work_dir.clear_runtime_networks(); - return Err(error); - } - let processes = match vm_config.config_qemu( - &work_dir, - &self.config.cvm, - &devices, - &runtime_networks, - ) { + Ok(configured) => configured, + Err(join_error) => { + self.release_vm_interfaces(&vm_config.manifest.id).await; + return Err(join_error).context("QEMU configuration task failed"); + } + }; + // Back in this future's hands, and held until the launch is done. + let _launch = launch; + let processes = match configured { Ok(processes) => processes, Err(error) => { - let _ = self - .remove_netd_networks(&vm_config.manifest.id, &runtime_networks) - .await; + self.release_vm_interfaces(&vm_config.manifest.id).await; return Err(error); } }; if let Err(error) = work_dir.set_runtime_networks(&runtime_networks) { - let _ = self - .remove_netd_networks(&vm_config.manifest.id, &runtime_networks) - .await; + self.release_vm_interfaces(&vm_config.manifest.id).await; return Err(error); } { @@ -504,17 +598,13 @@ impl App { } for process in processes { if let Err(err) = self.supervisor.deploy(&process).await { - if let Err(cleanup_error) = self - .remove_netd_networks(&vm_config.manifest.id, &runtime_networks) - .await - { - warn!(id, %cleanup_error, "failed to roll back netd-managed networking"); - } - if let Err(clear_err) = work_dir.clear_runtime_networks() { - warn!( - id, - "failed to clear runtime networks after start failure: {clear_err}" - ); + if self.release_vm_interfaces(&vm_config.manifest.id).await { + if let Err(clear_err) = work_dir.clear_runtime_networks() { + warn!( + id, + "failed to clear runtime networks after start failure: {clear_err}" + ); + } } if let Some(vm_state) = self.lock().get_mut(id) { vm_state.state.runtime_networks.clear(); @@ -539,13 +629,23 @@ impl App { } pub async fn stop_vm(&self, id: &str) -> Result<()> { + // Removal stops the VM itself and holds the launch lock while it does, + // so this would otherwise wait hours to do again what is already being + // done. + self.refuse_if_removing(id)?; if let Some(vm) = self.lock().get_mut(id) { vm.state.auto_restart.reset(); } + // Teardown deletes the same interfaces a launch creates, and derives + // their names rather than reading a record, so it must not overlap one. + let _launch = self.launch_lock(id).await; self.set_started(id, false)?; self.stop_vm_process(id).await?; - let networks = self.work_dir(id)?.runtime_networks(); - self.remove_netd_networks(id, &networks).await?; + // Not fallible: a VM that has been asked to stop is stopped whether or + // not netd could be reached. What is left behind keeps its marker and + // is released by `reconcile_network_cleanup`, the next launch, or the + // removal, whichever comes first. + self.release_vm_interfaces(id).await; Ok(()) } @@ -569,10 +669,32 @@ impl App { mapping.nic_index, ); } + + // Whatever an earlier boot left behind: a failed stop, a crash between + // creating an interface and using it, a NIC this VM no longer has, or + // a backend it no longer uses. Prepare replaces the names it is about + // to use, but only those, so an index nothing will claim again is only + // reachable from here. + // + // Before the early return: a VM that moved from a bridge to user-mode + // networking still has to give back what it held. + let vm_workdir = self.work_dir(&vm.manifest.id)?; + self.release_vm_interfaces(&vm.manifest.id).await; if !networks.iter().any(needs_netd_interface) { return Ok(()); } + // Still set only if the release above did not land, in which case netd + // may hold something from before this launch no matter how it ends. + let held_before_launch = vm_workdir.network_cleanup_pending(); + vm_workdir.mark_network_cleanup_pending()?; let qemu_uid = Uid::effective().as_raw(); + // Only ever read back out of a log line: netd is told where the VM + // lives so an operator holding an opaque TAP name can reach the VM + // without going through the VMM first. + let workdir = self + .work_dir(&vm.manifest.id) + .map(|dir| dir.path().display().to_string()) + .unwrap_or_default(); let mut prepared = Vec::new(); for (nic_index, network) in networks.iter_mut().enumerate() { if !needs_netd_interface(network) { @@ -604,6 +726,7 @@ impl App { // may not run libvirt at all. filtered, queues, + workdir: workdir.clone(), }) .await } @@ -615,6 +738,7 @@ impl App { qemu_uid, mode: network.macvtap_mode.clone(), queues, + workdir: workdir.clone(), }) .await } @@ -623,18 +747,28 @@ impl App { let response = match result { Ok(response) => response, Err(error) => { - // The client may have timed out while netd was still - // finishing this Prepare. Remove the in-flight identity - // first; netd runs operations in arrival order, so the - // removal runs after the Prepare completes. - if let Err(cleanup_error) = netd.remove_interface(identity.clone()).await { - warn!(%cleanup_error, "failed to roll back in-flight netd network"); + let unreachable = netd::is_unreachable(&error); + if unreachable && prepared.is_empty() && !held_before_launch { + // The first request of this launch never reached a + // netd, so none holds anything for this VM. Keeping + // the marker would make every later stop and removal + // wait for a netd this node may never run. + if let Err(clear_error) = vm_workdir.clear_network_cleanup_pending() { + warn!(%clear_error, "failed to clear pending network cleanup"); + } + } else { + // The client may have timed out while netd was still + // finishing this Prepare. Remove the in-flight identity + // first; netd runs operations in arrival order, so the + // removal runs after the Prepare completes. + if let Err(cleanup_error) = netd.remove_interface(identity.clone()).await { + warn!(%cleanup_error, "failed to roll back in-flight netd network"); + } + self.roll_back_prepared_networks(prepared).await; } - self.roll_back_prepared_networks(prepared).await; // netd's own message is about a socket or a TAP, so // neither the NIC that asked nor what a node has to install // to satisfy it appears anywhere in the failure. - let unreachable = netd::is_unreachable(&error); let mode = network.nic.mode.as_str(); let error = Err(error).context("failed to prepare netd-managed networking"); return if unreachable { @@ -708,8 +842,8 @@ impl App { /// snapshot started it. /// /// Guessing is all that is left, so guess the way that VMM would have, and - /// then write the guess down, so later teardown does not re-derive it from - /// node configuration that may by then have moved. + /// then write the guess down, so what is reported for this boot does not + /// drift with node configuration that may by then have moved. /// /// The way *that* VMM would have, not this one: a build old enough to leave /// no snapshot had no vhost and no multiqueue at all, so whatever this @@ -744,31 +878,66 @@ impl App { } } - /// Removes the host interfaces netd built for the NICs in `networks`. - pub(crate) async fn remove_netd_networks( - &self, - vm_id: &str, - networks: &[Networking], - ) -> Result<()> { - let netd = netd::client(&self.config.netd.socket); - let mut first_error = None; - for (nic_index, network) in networks.iter().enumerate().rev() { - if !needs_netd_interface(network) { - continue; - } - let identity = InterfaceIdentity { - instance_id: self.config.cvm.instance_id.clone(), - vm_id: vm_id.to_string(), - nic_index: nic_index as u32, - }; - if let Err(error) = netd.remove_interface(identity).await { - first_error.get_or_insert(error); + /// Releases every host interface netd holds for this VM. + /// + /// Gated on `.netd-pending` alone, never on the VM's current networking: + /// a VM whose NIC was a bridge when its TAP was built and is a user-mode + /// NIC now still holds that TAP. Non-fatal, because a VM must be able to + /// stop when netd cannot be reached; what a failed release leaves behind + /// keeps its marker, and [`App::reconcile_network_cleanup`], the next + /// launch or the removal retries it. + /// + /// Returns whether netd is known to hold nothing for this VM any more. + /// A removal reads that to decide whether it may delete the workdir: the + /// directory is what says to try again, and deleting it over a failed + /// release is what strands an interface with nothing left to reach it. + pub(crate) async fn release_vm_interfaces(&self, vm_id: &str) -> bool { + let workdir = match self.work_dir(vm_id) { + Ok(workdir) => workdir, + Err(error) => { + warn!(vm_id, %error, "cannot locate network cleanup state"); + return false; } + }; + if !workdir.network_cleanup_pending() { + return true; } - if let Some(error) = first_error { - return Err(error).context("failed to remove netd-managed networking"); + // Ask for the release, rather than asking whether it can be asked for. + // A probe first would put a second round trip in front of every stop + // and -- worse -- would make a *busy* netd look like an absent one and + // skip the release entirely. The operation itself cannot be misread + // that way: it succeeds, or it says netd is not there, or netd answers + // with a refusal. + let request = netd::VmRef { + instance_id: self.config.cvm.instance_id.clone(), + vm_id: vm_id.to_string(), + }; + match netd::client(&self.config.netd.socket) + .remove_vm(request) + .await + { + Ok(netd::RemoveVmResponse { removed }) => { + if let Err(error) = workdir.clear_network_cleanup_pending() { + warn!(vm_id, %error, "cannot clear pending network cleanup"); + return false; + } + if removed > 0 { + info!(vm_id, removed, "released netd-managed interfaces"); + } + true + } + Err(error) if netd::is_unreachable(&error) => { + debug!(vm_id, %error, "no netd to release interfaces from"); + false + } + Err(error) => { + warn!( + vm_id, + "failed to release netd-managed interfaces: {error:#}" + ); + false + } } - Ok(()) } pub(crate) async fn stop_vm_process(&self, id: &str) -> Result<()> { @@ -808,25 +977,26 @@ impl App { pub async fn remove_vm(&self, id: &str) -> Result<()> { { let mut state = self.lock(); - let vm = state.get_mut(id).context("VM not found")?; - if vm.state.removing { + state.get(id).context("VM not found")?; + if !state.start_removing(id) { // Already being removed — idempotent return Ok(()); } - vm.state.removing = true; } - // Persist the removing marker so crash recovery can resume - let work_dir = self.work_dir(id)?; - if let Err(err) = work_dir.set_removing() { - warn!("failed to write .removing marker for {id}: {err:?}"); - } + // Clears the in-memory mark if the removal cannot start. In particular, + // a removal that cannot persist its crash-recovery marker must not + // start: a crash would then forget it. + let mark = RemovalMark::new(self.clone(), id); + self.work_dir(id)? + .set_removing() + .with_context(|| format!("failed to write .removing marker for {id}"))?; // User-initiated removal always deletes the workdir let app = self.clone(); - let id = id.to_string(); tokio::spawn(async move { - if let Err(err) = app.finish_remove_vm(&id, true).await { + let id = mark.id.clone(); + if let Err(err) = app.finish_remove_vm(mark, true).await { error!("Background cleanup failed for {id}: {err:?}"); } }); @@ -838,14 +1008,30 @@ impl App { /// remove from supervisor, optionally delete workdir, and free CID. /// /// `delete_workdir`: true for user-initiated removal, false for orphan cleanup. - async fn finish_remove_vm(&self, id: &str, delete_workdir: bool) -> Result<()> { + /// + /// Owns the removal mark, which is cleared however this ends, including the + /// `?`s below and a panic in the task. + async fn finish_remove_vm(&self, mark: RemovalMark, delete_workdir: bool) -> Result<()> { + let id = mark.id.as_str(); + // Held across the stop, the wait and the release, not just the release. + // `removing` turns launches away, but a launch that passed that check + // before the marker was set is already inside the lock: it has not + // deployed yet, so the wait below sees nothing running and returns at + // once, and the release then deletes the interfaces of the QEMU that + // launch went on to start. Taking the lock first means the launch + // finishes before removal decides anything, and removal then stops what + // it actually started. + let _launch = self.launch_lock(id).await; // Stop the supervisor process (idempotent if already stopped) if let Err(err) = self.stop_vm_process(id).await { debug!("graceful VM stop during removal failed: {err:?}"); } - // Poll until the process is no longer running, then remove it. - // Some VMs take a long time to stop (e.g. 2+ hours), so we wait indefinitely. + // Poll until the process is no longer running, then remove it. The + // stop above is a SIGKILL, so this is however long the kernel takes to + // tear the VM down -- seconds for a large TD, unbounded for one wedged + // in a device reset. Waiting is still right: what follows deletes the + // interfaces and the workdir it is using. let mut poll_count: u64 = 0; loop { match self.supervisor.info(id).await { @@ -877,25 +1063,36 @@ impl App { } } - let runtime_networks = self.work_dir(id)?.runtime_networks(); - if let Err(error) = self.remove_netd_networks(id, &runtime_networks).await { - warn!(id, %error, "failed to remove netd-managed networking during VM removal"); - } - + let vm_path = self.work_dir(id)?; // Only delete the workdir for user-initiated removal or if .removing marker exists. // Orphaned supervisor processes without the marker keep their data intact. - let vm_path = self.work_dir(id)?; - if delete_workdir || vm_path.is_removing() { + if !(delete_workdir || vm_path.is_removing()) { + self.release_vm_interfaces(id).await; if vm_path.path().exists() { - if let Err(err) = fs::remove_dir_all(&vm_path) { - error!("failed to remove VM directory for {id}: {err:?}"); - } + info!( + "VM {id} workdir preserved (orphan cleanup): {}", + vm_path.path().display() + ); + } + } else { + // The directory holds `.netd-pending`, the only record of what netd + // may still hold for this VM, so it goes only once the release has + // landed. The release is idempotent: retry until it does. The VM + // stays marked as being removed meanwhile, so nothing can start it, + // and a repeated removal request is already in progress. + let mut delay = RELEASE_RETRY_INITIAL; + while !self.release_vm_interfaces(id).await { + warn!( + "VM {id} is waiting for netd to release its interfaces; retrying in {delay:?}" + ); + tokio::time::sleep(delay).await; + delay = (delay * 2).min(RELEASE_RETRY_MAX); + } + if let Err(err) = vm_path.remove_all() { + // `.removing` is deleted last, so what is left is resumed by + // the next reload. + error!("failed to remove VM directory for {id}: {err:?}"); } - } else if vm_path.path().exists() { - info!( - "VM {id} workdir preserved (orphan cleanup): {}", - vm_path.path().display() - ); } // Free CID and remove from memory (last step) @@ -915,22 +1112,20 @@ impl App { /// Returns false if a cleanup task is already running for this VM. fn spawn_finish_remove(&self, id: &str) -> bool { { - let mut state = self.lock(); - if let Some(vm) = state.get_mut(id) { - if vm.state.removing { - // Already being cleaned up — skip - return false; - } - vm.state.removing = true; + // Claimed in the set rather than in the entry: an orphaned + // supervisor process has no entry, and that is exactly the case + // where the launch lock is held with nothing turning waiters away. + if !self.lock().start_removing(id) { + // Already being cleaned up — skip + return false; } - // If VM is not in memory (e.g. orphaned supervisor process), no entry to guard - // but we still need to clean up the supervisor process. } + let mark = RemovalMark::new(self.clone(), id); let app = self.clone(); - let id = id.to_string(); tokio::spawn(async move { + let id = mark.id.clone(); // Don't pass delete_workdir=true; rely on .removing marker check inside - if let Err(err) = app.finish_remove_vm(&id, false).await { + if let Err(err) = app.finish_remove_vm(mark, false).await { error!("Background cleanup failed for {id}: {err:?}"); } }); @@ -1252,6 +1447,14 @@ impl App { }) .cloned() .collect::>(); + if !request.status.is_empty() { + infos.retain(|vm| match self.work_dir(&vm.config.manifest.id) { + Ok(work_dir) => { + vm.runtime_status(vms.get(&vm.config.manifest.id), &work_dir) == request.status + } + Err(_) => false, + }); + } infos.sort_by(|a, b| { a.config .manifest @@ -1503,6 +1706,72 @@ impl App { Ok(()) } + /// Releases what netd still holds for VMs that are not running. + /// + /// A stop whose release failed, or a VM that exited on its own, leaves + /// `.netd-pending` behind with nothing scheduled to act on it. Checking + /// here makes that state converge without waiting for the VM to be + /// started or removed. Costs a file check per VM when nothing is pending. + pub(crate) async fn reconcile_network_cleanup(&self) { + let ids: Vec = self.lock().vms.keys().cloned().collect(); + let pending: Vec = ids + .into_iter() + .filter(|id| { + self.work_dir(id) + .is_ok_and(|workdir| workdir.network_cleanup_pending()) + }) + .collect(); + if pending.is_empty() { + return; + } + // A running VM keeps its marker for as long as it runs. One listing + // skips those up front, so a pass does not queue behind every running + // VM's launch lock only to find it running. Each remaining VM is still + // checked again under its lock. + let running: HashSet = match self.supervisor.list().await { + Ok(processes) => processes + .into_iter() + .filter(|process| process.state.status.is_running()) + .map(|process| process.config.id) + .collect(), + Err(error) => { + debug!("skipping network cleanup: {error:#}"); + return; + } + }; + for id in pending { + if !running.contains(&id) { + self.reconcile_vm_network_cleanup(&id).await; + } + } + } + + async fn reconcile_vm_network_cleanup(&self, id: &str) { + let pending = self + .work_dir(id) + .is_ok_and(|workdir| workdir.network_cleanup_pending()); + // A removal releases the VM itself, and holds the launch lock for as + // long as that takes. + if !pending || self.refuse_if_removing(id).is_err() { + return; + } + // A launch holds this lock until QEMU is deployed, so the running + // check below cannot see a launch half done. + let _launch = self.launch_lock(id).await; + if self.refuse_if_removing(id).is_err() { + return; + } + match self.supervisor.info(id).await { + Ok(Some(info)) if info.state.status.is_running() => return, + Ok(_) => {} + Err(error) => { + debug!(id, "skipping network cleanup: {error:#}"); + return; + } + } + self.release_vm_interfaces(id).await; + } + pub(crate) async fn try_restart_exited_vms(&self) -> Result<()> { let running_vms = self .supervisor @@ -1995,6 +2264,418 @@ mod tests { use super::mr_config::{mr_config_version, MrConfigVersion}; use super::*; + fn test_app() -> App { + use rocket::figment::providers::Format as _; + let config: Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + App::new(config, SupervisorClient::new("http://127.0.0.1:0")) + } + + fn test_config(netd_socket: &Path, run_path: &Path) -> Config { + use rocket::figment::providers::Format as _; + let mut config: Config = rocket::figment::Figment::from( + rocket::figment::providers::Toml::string(crate::config::DEFAULT_CONFIG), + ) + .extract() + .unwrap(); + config.netd.socket = netd_socket.to_path_buf(); + config.cvm.instance_id = "test-instance".to_string(); + config.run_path = run_path.to_path_buf(); + config + } + + fn app_talking_to(netd_socket: &Path) -> App { + let run_path = netd_socket.parent().unwrap_or(Path::new("/nonexistent")); + App::new( + test_config(netd_socket, run_path), + SupervisorClient::new("http://127.0.0.1:0"), + ) + } + + /// A supervisor with no remaining process lets removal exercise the real + /// directory and netd paths without launching a VM. + async fn stopped_supervisor() -> (SupervisorClient, tokio::task::JoinHandle<()>) { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let task = tokio::spawn(async move { + loop { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut reader = BufReader::new(&mut stream); + let mut header = String::new(); + loop { + header.clear(); + assert!(reader.read_line(&mut header).await.unwrap() > 0); + if header == "\r\n" { + break; + } + } + let body = serde_json::to_string(&supervisor_client::supervisor::Response::Data( + Option::::None, + )) + .unwrap(); + stream + .write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body, + ) + .as_bytes(), + ) + .await + .unwrap(); + } + }); + (SupervisorClient::new(&format!("http://{address}")), task) + } + + /// A VM directory holding `.netd-pending`, as a launch leaves it. + fn pending_workdir(app: &App, id: &str) -> VmWorkDir { + let workdir = app.work_dir(id).unwrap(); + std::fs::create_dir_all(workdir.path()).unwrap(); + workdir.mark_network_cleanup_pending().unwrap(); + workdir + } + + /// A removal keeps retrying while netd is away, keeps the VM marked as + /// being removed so nothing can start it, and finishes on its own once netd + /// comes back -- without a VMM restart or another request. + #[tokio::test] + async fn a_removal_retries_until_netd_releases_and_then_finishes() { + let (supervisor, server) = stopped_supervisor().await; + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("netd.sock"); + let app = App::new(test_config(&socket, dir.path()), supervisor); + let workdir = pending_workdir(&app, "vm-1"); + workdir.set_removing().unwrap(); + + assert!(app.lock().start_removing("vm-1")); + let mark = RemovalMark::new(app.clone(), "vm-1"); + let removal = tokio::spawn({ + let app = app.clone(); + async move { app.finish_remove_vm(mark, true).await } + }); + + tokio::time::sleep(Duration::from_millis(200)).await; + assert!( + !removal.is_finished(), + "nothing released, so nothing removed" + ); + assert!(workdir.path().exists()); + assert!(workdir.network_cleanup_pending()); + assert!(app.refuse_if_removing("vm-1").is_err()); + // Asking again while it retries is the same removal. + assert!(!app.lock().start_removing("vm-1")); + + let netd = netd::testing::FakeNetd::spawn_at(&socket, &["RemoveVm"]); + tokio::time::timeout(Duration::from_secs(10), removal) + .await + .expect("the removal converges once netd answers") + .unwrap() + .unwrap(); + assert!(!workdir.path().exists()); + assert!(app.refuse_if_removing("vm-1").is_ok()); + assert!(netd.methods().iter().all(|method| method == "RemoveVm")); + server.abort(); + } + + /// A VM that never asked netd for an interface has nothing to wait for: + /// its removal completes with netd absent. + #[tokio::test] + async fn a_removal_with_nothing_pending_does_not_wait_for_netd() { + let (supervisor, server) = stopped_supervisor().await; + let dir = tempfile::tempdir().unwrap(); + let app = App::new( + test_config(&dir.path().join("absent-netd.sock"), dir.path()), + supervisor, + ); + let workdir = app.work_dir("user-only").unwrap(); + std::fs::create_dir_all(workdir.path()).unwrap(); + workdir.set_removing().unwrap(); + assert!(app.lock().start_removing("user-only")); + let mark = RemovalMark::new(app.clone(), "user-only"); + tokio::time::timeout(Duration::from_secs(10), app.finish_remove_vm(mark, true)) + .await + .unwrap() + .unwrap(); + assert!(!workdir.path().exists()); + server.abort(); + } + + #[tokio::test] + async fn successful_sweep_clears_the_pending_marker() { + let netd = netd::testing::FakeNetd::spawn(&["RemoveVm"]); + let dir = tempfile::tempdir().unwrap(); + let app = App::new( + test_config(netd.socket(), dir.path()), + SupervisorClient::new("http://127.0.0.1:0"), + ); + let workdir = pending_workdir(&app, "vm-1"); + assert!(app.release_vm_interfaces("vm-1").await); + assert!(!workdir.network_cleanup_pending()); + assert!(workdir.path().exists()); + } + + /// The marker is the only thing that says netd may hold something. Without + /// it there is nothing to release and nobody to ask, so a user-mode VM on a + /// node with no netd never waits on one. + #[tokio::test] + async fn a_vm_with_nothing_pending_never_asks_netd() { + let netd = netd::testing::FakeNetd::spawn(&["RemoveVm"]); + let app = app_talking_to(netd.socket()); + assert!(app.release_vm_interfaces("vm-1").await); + assert!(netd.methods().is_empty()); + } + + fn bridge_vm(app: &App, id: &str) -> (VmConfig, Vec) { + let mut manifest = test_manifest(2048); + manifest.id = id.to_string(); + manifest.networks = vec![NicNetworking { + mode: NetworkingMode::Bridge, + bridge: "dstack-br0".to_string(), + ..NicNetworking::default() + }]; + let networks = app.runtime_networks(&manifest); + let vm = VmConfig { + workdir: app.work_dir(id).unwrap().path().to_path_buf(), + manifest, + image: test_tdx_image(true), + cid: 3, + gateway_enabled: false, + }; + (vm, networks) + } + + /// A launch that never reached a netd built nothing, so it must not leave + /// a marker that makes every later stop and removal wait for one. What + /// an earlier boot may still hold keeps its marker. + #[tokio::test] + async fn a_launch_that_never_reached_netd_leaves_nothing_pending() { + let dir = tempfile::tempdir().unwrap(); + let app = App::new( + test_config(&dir.path().join("absent-netd.sock"), dir.path()), + SupervisorClient::new("http://127.0.0.1:0"), + ); + let (vm, mut networks) = bridge_vm(&app, "vm-1"); + let workdir = app.work_dir("vm-1").unwrap(); + std::fs::create_dir_all(workdir.path()).unwrap(); + + let error = app + .prepare_netd_networks(&vm, &mut networks) + .await + .unwrap_err(); + assert!( + format!("{error:#}").contains("run dstack-vmm netd"), + "{error:#}" + ); + assert!(!workdir.network_cleanup_pending()); + + workdir.mark_network_cleanup_pending().unwrap(); + app.prepare_netd_networks(&vm, &mut networks) + .await + .unwrap_err(); + assert!( + workdir.network_cleanup_pending(), + "an unreleased earlier boot is still pending" + ); + } + + /// A netd that answered, even with a refusal, may have built something: + /// the marker stays. + #[tokio::test] + async fn a_launch_netd_refused_stays_pending() { + let netd = netd::testing::FakeNetd::spawn(&[]); + let app = app_talking_to(netd.socket()); + let (vm, mut networks) = bridge_vm(&app, "vm-1"); + let workdir = app.work_dir("vm-1").unwrap(); + std::fs::create_dir_all(workdir.path()).unwrap(); + app.prepare_netd_networks(&vm, &mut networks) + .await + .unwrap_err(); + assert!(workdir.network_cleanup_pending()); + assert_eq!(netd.methods(), vec!["PrepareBridge", "RemoveInterface"]); + } + + /// A netd outage must not become a fleet that cannot be stopped. + #[tokio::test] + async fn a_stop_survives_a_netd_that_is_not_there() { + let dir = tempfile::tempdir().unwrap(); + let app = App::new( + test_config(Path::new("/nonexistent/dstack-netd.sock"), dir.path()), + SupervisorClient::new("http://127.0.0.1:0"), + ); + let workdir = pending_workdir(&app, "vm-1"); + // Returns rather than propagating, and keeps the marker for a retry. + assert!(!app.release_vm_interfaces("vm-1").await); + assert!(workdir.network_cleanup_pending()); + } + + /// A removal deletes the workdir, and the workdir is the only thing left + /// that says to retry. So the release has to say whether it landed: an + /// answer means netd holds nothing, and anything else -- a refusal, or a + /// netd that is not there to ask -- means it may still. + #[tokio::test] + async fn a_release_says_whether_netd_still_holds_anything() { + for (handles, released) in [(&["RemoveVm"][..], true), (&[][..], false)] { + let netd = netd::testing::FakeNetd::spawn(handles); + let app = app_talking_to(netd.socket()); + let workdir = pending_workdir(&app, "vm-1"); + assert_eq!(app.release_vm_interfaces("vm-1").await, released); + assert_eq!(workdir.network_cleanup_pending(), !released); + } + } + + /// One request, no question in front of it. A probe on the hot path is a + /// second round trip whose failure mode is silence: a netd too slow to + /// answer it reads as an absent one, and an absent netd's release is + /// skipped. + #[tokio::test] + async fn a_release_asks_netd_to_sweep_without_a_question_first() { + let netd = netd::testing::FakeNetd::spawn(&["RemoveVm"]); + let app = app_talking_to(netd.socket()); + pending_workdir(&app, "vm-1"); + app.release_vm_interfaces("vm-1").await; + + assert_eq!(netd.methods(), vec!["RemoveVm"]); + let sweep = &netd.seen()[0]; + assert_eq!(sweep["vm_id"], "vm-1"); + assert_eq!(sweep["instance_id"], "test-instance"); + // A sweep names no NIC: reaching the indices the caller can no longer + // name is the entire point. + assert!(sweep.get("nic_index").is_none()); + } + + /// A stop whose release failed leaves the marker with nothing scheduled to + /// act on it. Reconciliation releases it once netd answers, and leaves a + /// VM that is being removed to its removal. + #[tokio::test] + async fn reconciliation_releases_what_a_stopped_vm_still_holds() { + let (supervisor, server) = stopped_supervisor().await; + let netd = netd::testing::FakeNetd::spawn(&["RemoveVm"]); + let app = App::new( + test_config(netd.socket(), netd.socket().parent().unwrap()), + supervisor, + ); + let workdir = pending_workdir(&app, "vm-1"); + + assert!(app.lock().start_removing("vm-1")); + app.reconcile_vm_network_cleanup("vm-1").await; + assert!( + netd.methods().is_empty(), + "a removal owns this VM's release" + ); + drop(RemovalMark::new(app.clone(), "vm-1")); + + app.reconcile_vm_network_cleanup("vm-1").await; + assert_eq!(netd.methods(), vec!["RemoveVm"]); + assert!(!workdir.network_cleanup_pending()); + + // Nothing pending: nothing to ask. + app.reconcile_vm_network_cleanup("vm-1").await; + assert_eq!(netd.methods(), vec!["RemoveVm"]); + server.abort(); + } + + /// The orphan cleanup runs for an ID that never loaded, so a guard living + /// in the VM entry is not there when it holds the launch lock across the + /// whole teardown. Without the mark, `StartVm` on that ID waits the + /// teardown out with no error and no log. + #[tokio::test] + async fn a_removal_with_no_vm_entry_still_turns_operations_away() { + let app = test_app(); + assert!(app.refuse_if_removing("orphan").is_ok()); + assert!(app.lock().start_removing("orphan")); + assert!( + app.refuse_if_removing("orphan").is_err(), + "an orphan being removed is still a VM being removed" + ); + // And the same removal cannot be started twice. + assert!(!app.lock().start_removing("orphan")); + } + + /// An ordinary early error clears the in-memory mark so the caller can + /// retry rather than leaving a VM permanently inaccessible. + #[tokio::test] + async fn a_removal_that_gives_up_early_does_not_leave_the_vm_marked() { + let app = test_app(); + { + let _mark = RemovalMark::new(app.clone(), "vm-1"); + assert!(app.lock().start_removing("vm-1")); + assert!(app.refuse_if_removing("vm-1").is_err()); + } + assert!( + app.refuse_if_removing("vm-1").is_ok(), + "the mark is cleared however the removal ends" + ); + } + + /// A restart decided before a stop must not outlive it. The restart task + /// reads the started flag off disk and only then queues a launch, which + /// waits for the lock the stop is holding; without a re-read under that + /// lock the launch resurrects a VM the operator was told was stopped. + #[tokio::test] + async fn an_automatic_restart_does_not_outlive_the_stop_it_raced() { + let dir = tempfile::tempdir().unwrap(); + let app = App::new( + test_config(Path::new("/nonexistent/netd.sock"), dir.path()), + SupervisorClient::new("http://127.0.0.1:0"), + ); + let work_dir = app.work_dir("vm-1").unwrap(); + std::fs::create_dir_all(work_dir.path()).unwrap(); + + work_dir.set_started(false).unwrap(); + app.start_vm_with_restart_policy("vm-1", false) + .await + .expect("an automatic restart of a stopped VM does nothing"); + + // The flag is the whole difference: with it set, the same call goes on + // to do the work, and fails here for want of a VM to launch. + work_dir.set_started(true).unwrap(); + assert!(app + .start_vm_with_restart_policy("vm-1", false) + .await + .is_err()); + + // An explicit start sets the flag itself and has nothing to re-read, + // so it is never turned away by one. + work_dir.set_started(false).unwrap(); + assert!(app.start_vm("vm-1").await.is_err()); + } + + /// The window a launch spends between reading "not running" and actually + /// starting QEMU is long -- a GPU reset, a netd conversation -- and the + /// sweep inside it deletes interfaces by deriving their names. Two entrants + /// in that window meant the loser deleting the winner's live TAPs. + #[tokio::test] + async fn one_vm_launches_at_a_time_and_the_lock_map_stays_small() { + let app = test_app(); + let held = app.launch_lock("vm-1").await; + + // A different VM is never blocked by it: a slow start must not stall + // every other launch on the node. + let other = tokio::time::timeout(Duration::from_millis(50), app.launch_lock("vm-2")).await; + assert!(other.is_ok(), "an unrelated VM must not wait"); + + // The same VM is. + let same = tokio::time::timeout(Duration::from_millis(50), app.launch_lock("vm-1")).await; + assert!(same.is_err(), "a second entrant must wait for the first"); + + drop(held); + drop(other); + tokio::time::timeout(Duration::from_millis(50), app.launch_lock("vm-1")) + .await + .expect("the lock is released"); + + // Nothing is in flight now, so the map holds nothing either. + assert!(app.launch_locks.lock().unwrap().len() <= 1); + let _ = app.launch_lock("vm-3").await; + assert!(app.launch_locks.lock().unwrap().len() <= 2); + } + #[test] fn accepts_server_generated_ids() { validate_vm_id(&uuid::Uuid::new_v4().to_string()).unwrap(); @@ -2369,6 +3050,8 @@ mod tests { assert_eq!(manifest.networks.len(), 1); assert_eq!(manifest.networks[0].mode, NetworkingMode::Bridge); assert_eq!(manifest.networks[0].bridge, "dstack-br0"); + // A manifest written before the option existed keeps thin disks. + assert_eq!(manifest.disk_prealloc, DiskPrealloc::Off); } fn write_u16_le_at(buf: &mut [u8], off: usize, value: u16) { @@ -2465,6 +3148,7 @@ mod tests { swtpm: false, networks: vec![], volumes: vec![], + disk_prealloc: DiskPrealloc::Off, } } @@ -2819,6 +3503,7 @@ mod tests { swtpm: false, networks: vec![], volumes: vec![], + disk_prealloc: DiskPrealloc::Off, }; let mr_config = MrConfigV3::new( @@ -3094,6 +3779,14 @@ impl VmState { pub(crate) struct AppState { cid_pool: IdPool, vms: HashMap, + /// The VMs a removal is currently working on. + /// + /// Separate from `VmState::removing` because the set has to outlive the + /// entry. Orphan cleanup runs for IDs that never loaded into `vms`, and + /// `finish_remove_vm` holds the launch lock across the whole teardown, so + /// a guard that lives in the entry cannot turn away the operation that + /// would otherwise wait that teardown out. + removing: HashSet, } impl AppState { @@ -3116,6 +3809,47 @@ impl AppState { pub fn iter_vms(&self) -> impl Iterator { self.vms.values() } + + /// Claims `id` for a removal. False when one already has it. + fn start_removing(&mut self, id: &str) -> bool { + if let Some(vm) = self.vms.get_mut(id) { + vm.state.removing = true; + } + self.removing.insert(id.to_string()) + } + + fn is_removing(&self, id: &str) -> bool { + self.removing.contains(id) + } +} + +/// Clears the in-flight removal mark however the removal ends. +/// +/// `finish_remove_vm` returns early on more than its happy path, and a mark +/// left behind is not a stale flag: every operation on that VM answers "being +/// removed" from then on, including the removal that would retry. +struct RemovalMark { + app: App, + id: String, +} + +impl RemovalMark { + fn new(app: App, id: &str) -> Self { + Self { + app, + id: id.to_string(), + } + } +} + +impl Drop for RemovalMark { + fn drop(&mut self) { + let mut state = self.app.lock(); + state.removing.remove(&self.id); + if let Some(vm) = state.vms.get_mut(&self.id) { + vm.state.removing = false; + } + } } /// Reject VM ids that would escape `run_path` once joined into a filesystem diff --git a/dstack/vmm/src/app/image.rs b/dstack/vmm/src/app/image.rs index fcb5600f6..ae941fb74 100644 --- a/dstack/vmm/src/app/image.rs +++ b/dstack/vmm/src/app/image.rs @@ -10,11 +10,11 @@ use anyhow::{bail, Context, Result}; use dstack_types::{ version::Version, AwsOsImageMeasurementDocument, AwsPcrReplay, GcpOsImageMeasurementDocument, GcpTpmReplay, SevOsImageMeasurementDocument, TdxOsImageMeasurementDocument, - GCP_MEASUREMENT_FILENAME, SNP_MEASUREMENT_FILENAME, TDX_MEASUREMENT_FILENAME, + AWS_MEASUREMENT_FILENAME, GCP_MEASUREMENT_FILENAME, SNP_MEASUREMENT_FILENAME, + TDX_MEASUREMENT_FILENAME, }; use serde::{Deserialize, Serialize}; -const AWS_MEASUREMENT_FILENAME: &str = "measurement.aws.cbor"; const AWS_PCR_REPLAY_FILENAME: &str = "measurement.aws.replay.json"; const GCP_TPM_EVENT_LOG_FILENAME: &str = "measurement.gcp.eventlog.bin"; diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index f0652dd13..74e3a7958 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -17,7 +17,7 @@ use super::{ }; use crate::{ app::Manifest, - config::{CvmConfig, CvmPlatform, Networking, NetworkingMode, ProcessAnnotation}, + config::{CvmConfig, CvmPlatform, DiskPrealloc, Networking, NetworkingMode, ProcessAnnotation}, netd::{tap_name, InterfaceIdentity}, vm_launcher::{ChildCommand, LaunchSpec, OpenFile}, }; @@ -143,22 +143,51 @@ pub struct VmConfig { pub gateway_enabled: bool, } +/// Build the `qemu-img create` arguments for a CVM data disk. +fn qemu_img_create_args( + image_file: &Path, + backing_file: Option<&Path>, + size: &str, + prealloc: DiskPrealloc, +) -> Vec { + let mut args = vec!["create".to_string(), "-f".to_string(), "qcow2".to_string()]; + if let Some(backing_file) = backing_file { + args.push("-o".to_string()); + args.push(format!("backing_file={}", backing_file.display())); + args.push("-o".to_string()); + args.push("backing_fmt=qcow2".to_string()); + } + if !prealloc.is_off() { + // qcow2 rejects preallocation on top of a backing file unless + // subcluster allocation is on, because without it a preallocated + // cluster cannot record which of its parts still read through to the + // backing image. Images that ship a base hda always land here, so the + // choice is extended_l2 (QEMU 5.2+) or no preallocation at all. + if backing_file.is_some() { + args.push("-o".to_string()); + args.push("extended_l2=on".to_string()); + } + args.push("-o".to_string()); + args.push(format!("preallocation={}", prealloc.as_str())); + } + args.push(image_file.display().to_string()); + args.push(size.to_string()); + args +} + fn create_hd( image_file: impl AsRef, backing_file: Option>, size: &str, + prealloc: DiskPrealloc, ) -> Result<()> { - let mut command = Command::new("qemu-img"); - command.arg("create").arg("-f").arg("qcow2"); - if let Some(backing_file) = backing_file { - command - .arg("-o") - .arg(format!("backing_file={}", backing_file.as_ref().display())); - command.arg("-o").arg("backing_fmt=qcow2"); - } - command.arg(image_file.as_ref()); - command.arg(size); - let output = command.output()?; + let args = qemu_img_create_args( + image_file.as_ref(), + backing_file.as_ref().map(AsRef::as_ref), + size, + prealloc, + ); + let output = Command::new("qemu-img").args(&args).output()?; if !output.status.success() { bail!( "Failed to create disk: {}", @@ -241,7 +270,6 @@ impl PreparedQemuLaunch { networks: &[Networking], ) -> Result { let workdir = VmWorkDir::new(workdir); - prepare_data_disk(vm, &workdir)?; prepare_shared_dir(&workdir)?; let app_compose = workdir.app_compose().context("failed to get app compose")?; let platform = cfg.resolved_platform(); @@ -290,6 +318,10 @@ impl PreparedQemuLaunch { (None, None) }; prepare_shared_disk(&workdir, cfg)?; + // Last, because it is the one expensive step: a preallocating VM + // writes its whole disk here, and there is no point doing that only to + // fail on a missing swtpm binary or an unusable network. + prepare_data_disk(vm, &workdir)?; let tee_enabled = !vm.manifest.no_tee; let tdx_mr_config_id = if tee_enabled @@ -335,13 +367,37 @@ impl PreparedQemuLaunch { fn prepare_data_disk(vm: &VmConfig, workdir: &VmWorkDir) -> Result<()> { let hda_path = workdir.hda_path(); - if !hda_path.exists() { - create_hd( - &hda_path, - vm.image.hda.as_ref(), - &format!("{}G", vm.manifest.disk_size), - )?; + if hda_path.exists() { + return Ok(()); + } + let prealloc = vm.manifest.disk_prealloc; + // Build the disk under a temporary name and rename it into place. An + // existing hda.img is taken as a finished disk, and preallocation makes + // creation slow enough -- minutes for a large `full` disk -- that a VMM + // killed midway would otherwise leave a half-written image that the next + // start would boot from. + let partial_path = hda_path.with_extension("img.partial"); + if partial_path.exists() { + tracing::warn!( + "removing a leftover partial data disk: {}", + partial_path.display() + ); + fs::remove_file(&partial_path).context("failed to remove the partial data disk")?; } + if !prealloc.is_off() { + tracing::info!( + "creating {}GB data disk with preallocation={}", + vm.manifest.disk_size, + prealloc.as_str() + ); + } + create_hd( + &partial_path, + vm.image.hda.as_ref(), + &format!("{}G", vm.manifest.disk_size), + prealloc, + )?; + fs::rename(&partial_path, &hda_path).context("failed to publish the data disk")?; Ok(()) } @@ -1097,9 +1153,13 @@ mod tests { Figment, }; + use crate::config::DiskPrealloc; + use super::{ - amd_sev_snp_memory_backend_arg, macvtap_fd_layout, parse_amd_sev_snp_qmp_capabilities, - virtio_pci_device, PreparedQemuLaunch, PreparedVolume, QemuCommandBuilder, VmConfig, + amd_sev_snp_memory_backend_arg, create_hd, fs, macvtap_fd_layout, + parse_amd_sev_snp_qmp_capabilities, prepare_data_disk, qemu_img_create_args, + virtio_pci_device, Command, Path, PreparedQemuLaunch, PreparedVolume, QemuCommandBuilder, + VmConfig, }; use crate::app::image::{Image, ImageInfo}; use crate::app::{needs_swtpm, GpuConfig, GpuSpec, Manifest, PortMapping, VmVolume, VmWorkDir}; @@ -1157,6 +1217,136 @@ mod tests { ); } + #[test] + fn disk_creation_args_carry_preallocation_only_when_requested() { + let image = PathBuf::from("/vms/vm-1/hda.img"); + let backing = PathBuf::from("/images/base/hda.img"); + + assert_eq!( + qemu_img_create_args(&image, None, "20G", DiskPrealloc::Off), + vec!["create", "-f", "qcow2", "/vms/vm-1/hda.img", "20G"] + ); + + // No backing file: qcow2 takes preallocation on its own. + assert_eq!( + qemu_img_create_args(&image, None, "20G", DiskPrealloc::Falloc), + vec![ + "create", + "-f", + "qcow2", + "-o", + "preallocation=falloc", + "/vms/vm-1/hda.img", + "20G" + ] + ); + + // With a backing file, preallocation additionally needs extended_l2. + assert_eq!( + qemu_img_create_args(&image, Some(&backing), "20G", DiskPrealloc::Full), + vec![ + "create", + "-f", + "qcow2", + "-o", + "backing_file=/images/base/hda.img", + "-o", + "backing_fmt=qcow2", + "-o", + "extended_l2=on", + "-o", + "preallocation=full", + "/vms/vm-1/hda.img", + "20G" + ] + ); + + // An off disk keeps the exact arguments it had before the option + // existed, so existing deployments create byte-identical images. + assert_eq!( + qemu_img_create_args(&image, Some(&backing), "20G", DiskPrealloc::Off), + vec![ + "create", + "-f", + "qcow2", + "-o", + "backing_file=/images/base/hda.img", + "-o", + "backing_fmt=qcow2", + "/vms/vm-1/hda.img", + "20G" + ] + ); + } + + /// The option combination is the risky part: qemu-img rejects + /// preallocation on top of a backing file unless subcluster allocation is + /// on, and that rejection only shows up when qemu-img actually runs. + /// Skipped where qemu-img is not installed. + #[test] + fn preallocated_disk_reserves_host_space_on_top_of_a_backing_file() { + use std::os::unix::fs::MetadataExt; + + if Command::new("qemu-img").arg("--version").output().is_err() { + eprintln!("qemu-img not installed, skipping"); + return; + } + + let root = tempfile::TempDir::new().unwrap(); + let base = root.path().join("base.img"); + let hda = root.path().join("hda.img"); + create_hd(&base, None::<&Path>, "8M", DiskPrealloc::Off).unwrap(); + + create_hd(&hda, Some(&base), "8M", DiskPrealloc::Falloc).unwrap(); + let allocated = fs::metadata(&hda).unwrap().blocks() * 512; + assert!( + allocated >= 8 * 1024 * 1024, + "expected the 8M disk to be reserved, got {allocated} bytes" + ); + + fs::remove_file(&hda).unwrap(); + create_hd(&hda, Some(&base), "8M", DiskPrealloc::Off).unwrap(); + let allocated = fs::metadata(&hda).unwrap().blocks() * 512; + assert!( + allocated < 8 * 1024 * 1024, + "expected a thin disk, got {allocated} bytes" + ); + } + + #[test] + fn a_half_written_data_disk_is_never_taken_for_a_finished_one() { + if Command::new("qemu-img").arg("--version").output().is_err() { + eprintln!("qemu-img not installed, skipping"); + return; + } + + let root = tempfile::TempDir::new().unwrap(); + let workdir = VmWorkDir::new(root.path()); + let (_config, mut vm, _prepared) = test_launch_fixture(); + vm.manifest.disk_size = 1; + + // What a VMM killed during a slow preallocating create leaves behind. + let partial = workdir.hda_path().with_extension("img.partial"); + fs::write(&partial, b"half a qcow2").unwrap(); + + prepare_data_disk(&vm, &workdir).unwrap(); + assert!(workdir.hda_path().exists()); + assert!(!partial.exists(), "the stale partial disk was kept"); + let info = Command::new("qemu-img") + .args(["info", &workdir.hda_path().display().to_string()]) + .output() + .unwrap(); + assert!( + info.status.success(), + "the published disk is not a valid image" + ); + + // A finished disk is left exactly as it is. + let before = fs::metadata(workdir.hda_path()).unwrap().len(); + prepare_data_disk(&vm, &workdir).unwrap(); + assert_eq!(fs::metadata(workdir.hda_path()).unwrap().len(), before); + } + /// Minimal launch fixture. Nothing it points at has to exist on disk; every /// test overrides the fields it asserts on. fn test_launch_fixture() -> (Config, VmConfig, PreparedQemuLaunch) { @@ -1196,6 +1386,7 @@ mod tests { volumes: vec![VmVolume { source: "/does-not-exist/volume.img".into(), }], + disk_prealloc: DiskPrealloc::Off, }, image: Image { info: ImageInfo { diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index 48af6f864..cdd4f18e4 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -126,6 +126,19 @@ fn sanitize_optional>(value: Option) -> Option { value.filter(|value| !value.as_ref().trim().is_empty()) } +pub(crate) fn vm_runtime_status(removing: bool, started: bool, is_running: bool) -> &'static str { + if removing { + "removing" + } else { + match (started, is_running) { + (true, true) => "running", + (true, false) => "exited", + (false, true) => "stopping", + (false, false) => "stopped", + } + } +} + impl VmInfo { /// Takes no `CvmConfig` on purpose. Everything it reports about a VM's /// data plane was decided when that VM launched and written into @@ -206,6 +219,7 @@ impl VmInfo { }) .collect(), app_id: Some(self.manifest.app_id.clone()), + disk_prealloc: Some(self.manifest.disk_prealloc.as_str().to_string()), hugepages: self.manifest.hugepages, pin_numa: self.manifest.pin_numa, gpus: self.manifest.gpus.as_ref().map(|config| pb::GpuConfig { @@ -271,6 +285,18 @@ fn app_url(id: &str, custom_gateway_urls: &[String], gateway: &GatewayConfig) -> } impl VmState { + pub(crate) fn runtime_status( + &self, + process: Option<&ProcessInfo>, + workdir: &VmWorkDir, + ) -> &'static str { + vm_runtime_status( + self.state.removing, + workdir.started().unwrap_or(false), + process.is_some_and(|info| info.state.status.is_running()), + ) + } + pub fn merged_info(&self, process: Option<&ProcessInfo>, workdir: &VmWorkDir) -> VmInfo { fn truncate(duration: Duration) -> Duration { Duration::from_secs(duration.as_secs()) @@ -287,17 +313,7 @@ impl VmState { } let is_running = process.is_some_and(|info| info.state.status.is_running()); - let started = workdir.started().unwrap_or(false); - let status = if self.state.removing { - "removing" - } else { - match (started, is_running) { - (true, true) => "running", - (true, false) => "exited", - (false, true) => "stopping", - (false, false) => "stopped", - } - }; + let status = self.runtime_status(process, workdir); let uptime = display_timestamp(process.and_then(|info| info.state.started_at.as_ref())); let exited_at = display_timestamp(process.and_then(|info| info.state.stopped_at.as_ref())); let instance_id = sanitize_optional( @@ -328,9 +344,19 @@ impl VmState { #[cfg(test)] mod tests { - use super::{interfaces_to_proto, networking_to_proto, sanitize_optional}; + use super::{interfaces_to_proto, networking_to_proto, sanitize_optional, vm_runtime_status}; use crate::config::{NetworkingMode, NicNetworking}; + #[test] + fn runtime_status_covers_lifecycle() { + assert_eq!(vm_runtime_status(true, true, true), "removing"); + assert_eq!(vm_runtime_status(true, false, false), "removing"); + assert_eq!(vm_runtime_status(false, true, true), "running"); + assert_eq!(vm_runtime_status(false, true, false), "exited"); + assert_eq!(vm_runtime_status(false, false, true), "stopping"); + assert_eq!(vm_runtime_status(false, false, false), "stopped"); + } + /// Custom mode hands the operator the whole netdev string and the VMM never /// parses it, so it has no data-plane state to report. Reporting the /// resolved defaults instead asserted "vhost off, one queue" over a netdev diff --git a/dstack/vmm/src/app/workdir.rs b/dstack/vmm/src/app/workdir.rs index a9822c7ec..fd67072f3 100644 --- a/dstack/vmm/src/app/workdir.rs +++ b/dstack/vmm/src/app/workdir.rs @@ -143,6 +143,30 @@ impl VmWorkDir { self.workdir.join("runtime-networks.json") } + /// The only record that netd may hold host interfaces for this VM. + /// + /// Written durably before asking netd to create anything, and removed only + /// after a successful whole-VM sweep, so it is present whenever netd holds + /// something. The reverse need not hold: host interfaces do not survive a + /// reboot while this file does, and a marker with nothing behind it costs + /// one sweep. + pub fn mark_network_cleanup_pending(&self) -> Result<()> { + safe_write::safe_write(self.workdir.join(".netd-pending"), b"") + .context("failed to persist pending network cleanup") + } + + pub fn network_cleanup_pending(&self) -> bool { + self.workdir.join(".netd-pending").exists() + } + + pub fn clear_network_cleanup_pending(&self) -> Result<()> { + match fs::remove_file(self.workdir.join(".netd-pending")) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).context("failed to clear pending network cleanup"), + } + } + pub fn runtime_networks(&self) -> Vec { fs::read_to_string(self.runtime_networks_path()) .ok() @@ -212,7 +236,48 @@ impl VmWorkDir { } pub fn set_removing(&self) -> Result<()> { - fs::write(self.removing_marker(), "").context("failed to write .removing marker") + // Durable: this is what makes a removal resume after a crash. + safe_write::safe_write(self.removing_marker(), b"") + .context("failed to write .removing marker") + } + + /// Deletes the directory, `.removing` last. + /// + /// A deletion interrupted part-way leaves the marker behind, so the next + /// VMM start finishes the removal instead of loading what is left as a VM. + pub fn remove_all(&self) -> Result<()> { + let marker = self.removing_marker(); + let entries = match fs::read_dir(&self.workdir) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + for entry in entries { + let path = entry?.path(); + if path == marker { + continue; + } + if path.symlink_metadata()?.is_dir() { + fs::remove_dir_all(&path)?; + } else { + fs::remove_file(&path)?; + } + } + match fs::remove_file(&marker) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + if let Err(error) = fs::remove_dir(&self.workdir) { + // Something wrote into the directory while it was being emptied. + // Put the marker back, so what is left is still a removal to the + // next reload rather than a VM that fails to load. + if self.workdir.exists() { + let _ = self.set_removing(); + } + return Err(error.into()); + } + Ok(()) } pub fn path(&self) -> &Path { @@ -309,4 +374,18 @@ mod tests { fs::remove_dir_all(temp)?; Ok(()) } + + /// `.removing` goes last, so an interrupted deletion is still a removal. + #[test] + fn remove_all_deletes_everything_and_tolerates_a_missing_directory() { + let dir = tempfile::tempdir().unwrap(); + let workdir = VmWorkDir::new(dir.path().join("vm")); + std::fs::create_dir_all(workdir.path().join("shared")).unwrap(); + std::fs::write(workdir.path().join("shared/file"), b"x").unwrap(); + workdir.mark_network_cleanup_pending().unwrap(); + workdir.set_removing().unwrap(); + workdir.remove_all().unwrap(); + assert!(!workdir.path().exists()); + workdir.remove_all().unwrap(); + } } diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 49f013f57..23684737c 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -109,6 +109,63 @@ impl Protocol { } } +/// Host-side preallocation policy for a CVM data disk. The variants map +/// one-to-one onto `qemu-img`'s `preallocation=` option, so the semantics are +/// qcow2's, not ours. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum DiskPrealloc { + /// Allocate nothing up front. The image grows as the guest writes, so the + /// host may be oversubscribed and a guest write can fail with ENOSPC. + #[default] + Off, + /// Allocate the qcow2 metadata only. Cheap, and it removes the metadata + /// growth cost on first write, but it reserves no space for guest data. + Metadata, + /// Reserve the full disk with fallocate(2). Fast, and it moves the ENOSPC + /// failure to VM creation time where the operator sees it. + Falloc, + /// Reserve the full disk and write it out. Slowest to create -- a 1 TB + /// disk writes 1 TB of zeros -- but it leaves nothing to allocate later. + Full, +} + +impl FromStr for DiskPrealloc { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + Ok(match s { + "off" => DiskPrealloc::Off, + "metadata" => DiskPrealloc::Metadata, + "falloc" => DiskPrealloc::Falloc, + "full" => DiskPrealloc::Full, + _ => bail!("invalid disk preallocation mode: {s}"), + }) + } +} + +impl DiskPrealloc { + pub fn as_str(&self) -> &'static str { + match self { + DiskPrealloc::Off => "off", + DiskPrealloc::Metadata => "metadata", + DiskPrealloc::Falloc => "falloc", + DiskPrealloc::Full => "full", + } + } + + pub fn is_off(&self) -> bool { + matches!(self, DiskPrealloc::Off) + } + + /// Whether the mode reserves host blocks for guest data. `metadata` does + /// not -- it only builds the qcow2 tables -- so a guest that discards + /// takes nothing away from it. + pub fn reserves_data_blocks(&self) -> bool { + matches!(self, DiskPrealloc::Falloc | DiskPrealloc::Full) + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] pub enum CvmPlatform { @@ -438,6 +495,14 @@ pub struct CvmConfig { /// Host sharing mode. (9p, vhd, vvfat) pub host_share_mode: String, + /// Default preallocation for newly created CVM data disks. A deploy + /// request may pick another mode per VM. This is a host storage policy: + /// the guest cannot observe it and it never enters the measurements. + /// It applies to disks created from here on; a VM that already has its + /// data disk keeps whatever it was created with. + #[serde(default)] + pub disk_prealloc: DiskPrealloc, + /// QGS (Quote Generation Service) vsock port for kernel-level TSM support. /// When set, QEMU will pass this port to tdx-guest for configfs-tsm quote generation. /// The guest kernel will use this vsock port to communicate with the host QGS. @@ -1283,6 +1348,27 @@ impl Config { mod tests { use super::*; + /// The four names are spelled out three times -- serde, `FromStr` and + /// `as_str` -- so a new variant that misses one of them fails here. + #[test] + fn disk_prealloc_names_round_trip_through_every_encoding() { + for mode in [ + DiskPrealloc::Off, + DiskPrealloc::Metadata, + DiskPrealloc::Falloc, + DiskPrealloc::Full, + ] { + let name = mode.as_str(); + assert_eq!(name.parse::().unwrap(), mode); + assert_eq!(serde_json::to_string(&mode).unwrap(), format!("\"{name}\"")); + assert_eq!( + serde_json::from_str::(&format!("\"{name}\"")).unwrap(), + mode + ); + } + assert!("sparse".parse::().is_err()); + } + #[cfg(unix)] fn cvm_with_qemu(dir: &tempfile::TempDir, script: &str) -> CvmConfig { use std::os::unix::fs::PermissionsExt; diff --git a/dstack/vmm/src/gpu_reset.rs b/dstack/vmm/src/gpu_reset.rs index e58876b9a..5e76404d0 100644 --- a/dstack/vmm/src/gpu_reset.rs +++ b/dstack/vmm/src/gpu_reset.rs @@ -2,25 +2,49 @@ // // SPDX-License-Identifier: Apache-2.0 +//! GPU sanitization before QEMU attach. +//! +//! Terminating QEMU while GSP/SPDM initialization is in progress can leave +//! stale FSP/SPDM session state in a GPU. VFIO's attach-time FLR does not +//! clear that state; a PCIe Secondary Bus Reset does. The next guest +//! otherwise fails with an SPDM timeout followed by GSP/RmInitAdapter errors. +//! +//! The VMM runs as an unprivileged user, so it cannot issue the SBR by +//! writing Bridge Control in the upstream bridge's sysfs config space: that +//! file is writable by root only, as is /sys/bus/pci/drivers_probe. Instead +//! the VFIO_DEVICE_PCI_HOT_RESET ioctl asks the kernel to perform the same +//! bus reset. The ioctl is authorized by device ownership rather than +//! privilege: the caller must present an fd for every VFIO group affected by +//! the reset. The group nodes under /dev/vfio are the same ones QEMU opens +//! to attach the GPU, so the VMM user already has access to them. +//! +//! A single group fd suffices only because every sanitized GPU sits alone +//! behind a dedicated PCIe bridge and alone in its IOMMU group. The bridge +//! topology is validated first, and the kernel-reported set of devices +//! affected by the reset must all belong to the GPU's own group; anything +//! else aborts the launch rather than risking disruption to other devices. +//! +//! A VFIO group can be opened by only one process at a time, so every fd is +//! closed again before QEMU is spawned. + use std::{ collections::BTreeSet, - fs::{File, OpenOptions}, + fs::File, os::unix::fs::FileExt, path::{Path, PathBuf}, + sync::Arc, thread, time::{Duration, Instant}, }; use anyhow::{bail, Context, Result}; use tracing::info; +use vfio_ioctls::{PciHotResetDevice, VfioContainer, VfioDevice}; use crate::{app::GpuConfig, config::GpuConfig as HostGpuConfig}; const PCI_SYSFS_DEVICES: &str = "/sys/bus/pci/devices"; const PCI_BRIDGE_CLASS: u32 = 0x0604; -const PCI_BRIDGE_CONTROL: u64 = 0x3e; -const PCI_BRIDGE_CTL_BUS_RESET: u16 = 1 << 6; -const SBR_ASSERT_TIME: Duration = Duration::from_millis(100); const SBR_POLL_INTERVAL: Duration = Duration::from_millis(100); const SBR_STABLE_TIME: Duration = Duration::from_millis(500); @@ -33,63 +57,123 @@ pub fn sanitize_on_attach(host: &HostGpuConfig, devices: &GpuConfig) -> Result<( if !host.enabled || !host.sanitize_on_attach || devices.gpus.is_empty() { return Ok(()); } - sanitize_at( - Path::new(PCI_SYSFS_DEVICES), - devices, - Duration::from_millis(host.sbr_timeout_ms), - ) + let slots = devices + .gpus + .iter() + .map(|gpu| gpu.slot.clone()) + .collect::>(); + sanitize_slots(&slots, Duration::from_millis(host.sbr_timeout_ms)) } -fn sanitize_at(sysfs_devices: &Path, devices: &GpuConfig, timeout: Duration) -> Result<()> { - let selected = devices - .gpus +/// Sanitizes the given GPU slots. Entry point for the `sanitize-gpu` +/// subcommand; `sanitize_on_attach` funnels here as well. +pub fn sanitize_slots(slots: &[String], timeout: Duration) -> Result<()> { + if slots.is_empty() { + bail!("no GPU slots specified"); + } + let selected = slots .iter() - .map(|gpu| normalize_slot(&gpu.slot)) + .map(|slot| normalize_slot(slot)) .collect::>(); - let mut bridges = BTreeSet::new(); + sanitize_at(Path::new(PCI_SYSFS_DEVICES), &selected, timeout) +} - for gpu in &selected { +fn sanitize_at(sysfs_devices: &Path, selected: &BTreeSet, timeout: Duration) -> Result<()> { + for gpu in selected { let bridge = upstream_bridge(sysfs_devices, gpu)?; ensure_dedicated_bridge(&bridge, gpu)?; - bridges.insert(bridge); } - for bridge in bridges { - info!(bridge = %bridge.display(), "sanitizing GPU with PCIe Secondary Bus Reset"); - secondary_bus_reset(&bridge) - .with_context(|| format!("failed to sanitize GPU using bridge {}", bridge.display()))?; + for gpu in selected { + info!(gpu = %gpu, "sanitizing GPU with VFIO PCI hot reset"); + hot_reset(sysfs_devices, gpu) + .with_context(|| format!("failed to sanitize GPU {gpu} with VFIO hot reset"))?; + } + wait_for_vfio_ready(sysfs_devices, selected, timeout)?; + Ok(()) +} + +/// Issues a Secondary Bus Reset on the GPU's upstream bridge through VFIO. +/// +/// All fds are dropped on return so that QEMU can open the group afterwards. +fn hot_reset(sysfs_devices: &Path, slot: &str) -> Result<()> { + let group_id = iommu_group_id(sysfs_devices, slot)?; + let container = Arc::new(VfioContainer::new(None).context("failed to open VFIO container")?); + let device = VfioDevice::new(&sysfs_devices.join(slot), container) + .with_context(|| format!("failed to open VFIO device {slot}"))?; + + let dependents = device + .pci_hot_reset_info() + .context("failed to query PCI hot reset scope")?; + let foreign = dependents + .iter() + .filter(|dep| dep.group_id != group_id) + .map(format_dependent) + .collect::>(); + if !foreign.is_empty() { + bail!( + "refusing hot reset of GPU {slot}: it would also reset devices outside \ + IOMMU group {group_id}: {}", + foreign.join(", ") + ); } - wait_for_vfio_ready(sysfs_devices, &selected, timeout)?; + info!( + gpu = %slot, + affected = %dependents.iter().map(format_dependent).collect::>().join(", "), + "issuing VFIO PCI hot reset" + ); + + device + .pci_hot_reset_own_group() + .context("failed to perform PCI hot reset")?; Ok(()) } +fn format_dependent(dep: &PciHotResetDevice) -> String { + format!( + "{:04x}:{:02x}:{:02x}.{:x} (group {})", + dep.segment, + dep.bus, + dep.devfn >> 3, + dep.devfn & 0x7, + dep.group_id + ) +} + +fn iommu_group_id(sysfs_devices: &Path, slot: &str) -> Result { + let link = sysfs_devices.join(slot).join("iommu_group"); + let group = link + .canonicalize() + .with_context(|| format!("failed to resolve IOMMU group of {slot}"))?; + group + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| name.parse().ok()) + .with_context(|| format!("invalid IOMMU group path {}", group.display())) +} + fn wait_for_vfio_ready( sysfs_devices: &Path, devices: &BTreeSet, timeout: Duration, ) -> Result<()> { - let drivers_probe = sysfs_devices - .parent() - .context("PCI sysfs devices directory has no parent")? - .join("drivers_probe"); - let deadline = Instant::now() + timeout; + let started = Instant::now(); + let deadline = started + timeout; let mut stable_since = None; loop { let not_ready = devices .iter() - .filter(|slot| { - let device = sysfs_devices.join(slot); - if !driver_is_vfio(&device) { - let _ = fs_err::write(&drivers_probe, format!("{slot}\n")); - } - !is_vfio_ready(&device) - }) + .filter(|slot| !is_vfio_ready(&sysfs_devices.join(slot))) .cloned() .collect::>(); if not_ready.is_empty() { let since = *stable_since.get_or_insert_with(Instant::now); if since.elapsed() >= SBR_STABLE_TIME { - info!(count = devices.len(), "all sanitized GPUs are VFIO-ready"); + info!( + count = devices.len(), + elapsed_ms = started.elapsed().as_millis() as u64, + "all sanitized GPUs are VFIO-ready" + ); return Ok(()); } } else { @@ -194,39 +278,12 @@ fn is_pci_slot(name: &str) -> bool { .all(|(i, c)| matches!(i, 4 | 7 | 10) || c.is_ascii_hexdigit()) } -fn secondary_bus_reset(bridge: &Path) -> Result<()> { - let config_path = bridge.join("config"); - let config = OpenOptions::new() - .read(true) - .write(true) - .open(&config_path) - .with_context(|| format!("failed to open {}", config_path.display()))?; - let original = read_u16(&config, PCI_BRIDGE_CONTROL)?; - write_u16( - &config, - PCI_BRIDGE_CONTROL, - original | PCI_BRIDGE_CTL_BUS_RESET, - )?; - thread::sleep(SBR_ASSERT_TIME); - write_u16( - &config, - PCI_BRIDGE_CONTROL, - original & !PCI_BRIDGE_CTL_BUS_RESET, - )?; - Ok(()) -} - fn read_u16(file: &File, offset: u64) -> Result { let mut value = [0_u8; 2]; file.read_exact_at(&mut value, offset)?; Ok(u16::from_le_bytes(value)) } -fn write_u16(file: &File, offset: u64, value: u16) -> Result<()> { - file.write_all_at(&value.to_le_bytes(), offset)?; - Ok(()) -} - #[cfg(test)] mod tests { use super::*; @@ -247,6 +304,17 @@ mod tests { assert!(!is_pci_slot("0000:0g:00.0")); } + #[test] + fn formats_dependent_devices_with_pci_slot_and_function() { + let dep = PciHotResetDevice { + group_id: 46, + segment: 0, + bus: 0x0f, + devfn: 0x1, + }; + assert_eq!(format_dependent(&dep), "0000:0f:00.1 (group 46)"); + } + #[test] fn skips_sanitization_when_gpu_passthrough_is_disabled() { let host = HostGpuConfig { diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index 17b93d194..70dc8dc38 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -65,6 +65,8 @@ enum Command { CheckConfig, /// One-shot VM execution mode for debugging Run(RunArgs), + /// Sanitize GPUs with a VFIO PCI hot reset (debugging/ops) + SanitizeGpu(SanitizeGpuArgs), /// Run the privileged TAP and libvirt nwfilter broker. Netd(NetdArgs), /// Internal per-VM QEMU/swtpm launcher. @@ -77,6 +79,47 @@ struct NetdArgs { /// Override the Unix socket configured in [netd]. #[arg(long)] socket: Option, + /// Inspect a running netd instead of starting one. + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand)] +enum NetdCommand { + /// List every host interface netd holds. + /// + /// Answers the question a leak is made of -- whose is this interface -- + /// which deriving a name from an identity cannot. + List { + /// Only this VMM instance's interfaces. Defaults to every one netd + /// owns, including those it cannot attribute. + #[arg(long)] + instance: Option, + }, + /// Delete one interface by name. + /// + /// For what nothing else can name: an interface built before netd recorded + /// ownership, or by another netd, whose VM is gone. `netd list` shows these + /// with no instance and no VM, so nothing can derive the sweep that would + /// take them; an operator who can tell what they are says so here. + RemoveInterface { + /// The interface name, as `netd list` prints it. + name: String, + }, + /// Delete every interface netd holds for one VM. + /// + /// For a VM whose VMM will never ask again -- one whose directory was + /// deleted by hand, or whose instance is gone. A VMM retries the sweep for + /// its own VMs until it lands; this is for when no VMM will ever run it. + RemoveVm { + /// The `cvm.instance_id` of the VMM that created them. `netd list` + /// shows it. + #[arg(long)] + instance: String, + /// The VM's ID. + #[arg(long)] + vm: String, + }, } #[derive(ClapArgs)] @@ -91,6 +134,16 @@ struct RunArgs { dry_run: bool, } +#[derive(ClapArgs)] +struct SanitizeGpuArgs { + /// PCI slots of the GPUs to reset, e.g. 0000:0f:00.0 + #[arg(required = true)] + slots: Vec, + /// Maximum time in milliseconds for the GPUs to become VFIO-ready again + #[arg(long, default_value_t = 10_000)] + timeout_ms: u64, +} + #[derive(ClapArgs)] struct VmLauncherArgs { /// Path to the generated VM launch specification. @@ -176,6 +229,16 @@ async fn auto_restart_task(app: App) { } } +async fn network_cleanup_task(app: App) { + let mut interval = tokio::time::interval(Duration::from_secs(60)); + // A pass can outlast the period while netd is slow; do not burst after it. + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + interval.tick().await; + app.reconcile_network_cleanup().await; + } +} + async fn log_rotation_task(app: App) { if app.config.cvm.log.max_bytes == 0 { info!("Log rotation is disabled"); @@ -192,6 +255,74 @@ async fn log_rotation_task(app: App) { } } +/// Client-side netd subcommands. Talks to the socket like the VMM does, so it +/// needs whatever the socket's permissions ask for and not root. +async fn run_netd_command(config: &NetdConfig, command: &NetdCommand) -> Result<()> { + match command { + NetdCommand::List { instance } => { + let interfaces = netd::client(&config.socket) + .list_interfaces(netd::ListInterfacesRequest { + instance_id: instance.clone().unwrap_or_default(), + }) + .await + .context("failed to list netd interfaces")? + .interfaces; + println!( + "{:<16} {:<8} {:<24} {:<38} {:>3}", + "INTERFACE", "KIND", "INSTANCE", "VM", "NIC" + ); + let mut unattributed = 0; + for record in &interfaces { + if record.instance_id.is_none() { + unattributed += 1; + } + println!( + "{:<16} {:<8} {:<24} {:<38} {:>3}", + record.tap, + record.kind, + record.instance_id.as_deref().unwrap_or("-"), + record.vm_id.as_deref().unwrap_or("-"), + record + .nic_index + .map_or_else(|| "-".to_string(), |index| index.to_string()), + ); + } + println!(); + println!("{} interface(s)", interfaces.len()); + if unattributed > 0 { + // Not a fault to fix by hand: an interface built before netd + // recorded ownership, or by another netd, carries no record and + // gets one the next time its VM launches. + println!( + "{unattributed} carry no ownership record; `netd remove-interface` takes \ + one by name" + ); + } + Ok(()) + } + NetdCommand::RemoveInterface { name } => { + netd::client(&config.socket) + .remove_interface_by_name(netd::InterfaceName { tap: name.clone() }) + .await + .with_context(|| format!("failed to remove {name}"))?; + println!("removed {name}"); + Ok(()) + } + NetdCommand::RemoveVm { instance, vm } => { + let removed = netd::client(&config.socket) + .remove_vm(netd::VmRef { + instance_id: instance.clone(), + vm_id: vm.clone(), + }) + .await + .context("failed to remove the VM's interfaces")? + .removed; + println!("removed {removed} interface(s) for {vm}"); + Ok(()) + } + } +} + #[rocket::main] async fn main() -> Result<()> { { @@ -208,6 +339,14 @@ async fn main() -> Result<()> { return vm_launcher::run(Path::new(&launcher_args.spec)).await; } + // Needs no server configuration; only /dev/vfio access. + if let Some(Command::SanitizeGpu(sanitize_args)) = &args.command { + return gpu_reset::sanitize_slots( + &sanitize_args.slots, + Duration::from_millis(sanitize_args.timeout_ms), + ); + } + let figment = config::load_config_figment(args.config.as_deref()); if let Some(Command::Netd(netd_args)) = &args.command { let mut netd_config: NetdConfig = figment @@ -235,6 +374,9 @@ async fn main() -> Result<()> { .context("failed to load [cvm.network_filter] for netd")?, ); } + if let Some(command) = &netd_args.command { + return run_netd_command(&netd_config, command).await; + } return netd::serve(netd_config).await; } @@ -246,6 +388,7 @@ async fn main() -> Result<()> { // Preserve the existing startup validation. The broader static checks are // opt-in through `check-config` until they have seen wider deployment use. + netd::validate_instance_id(&config.cvm.instance_id)?; config .host_api .validate() @@ -271,6 +414,7 @@ async fn main() -> Result<()> { return Ok(()); } Command::Netd(_) => unreachable!("netd mode handled before server startup"), + Command::SanitizeGpu(_) => unreachable!("sanitize-gpu handled before config loading"), Command::Run(run_args) => { // One-shot VM execution mode return one_shot::run_one_shot( @@ -355,6 +499,7 @@ async fn main() -> Result<()> { state.reload_vms().await.context("Failed to reload VMs")?; tokio::spawn(auto_restart_task(state.clone())); tokio::spawn(log_rotation_task(state.clone())); + tokio::spawn(network_cleanup_task(state.clone())); tokio::select! { result = run_external_api(state.clone(), figment.clone(), api_auth) => { diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 7d0c7099b..82501b11c 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -29,7 +29,7 @@ use crate::app::{ validate_resolved_networks, App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir, }; -use crate::config::{CvmConfig, Networking, NetworkingMode, NicNetworking}; +use crate::config::{CvmConfig, DiskPrealloc, Networking, NetworkingMode, NicNetworking}; fn hex_sha256(data: &str) -> String { use sha2::Digest; @@ -312,6 +312,8 @@ pub fn create_manifest_from_vm_config( } let key_provider = key_provider_from_compose(&request.compose_file)?; let swtpm = needs_swtpm(key_provider, simulated_tee); + let disk_prealloc = disk_prealloc_from_vm_config(&request, cvm_config)?; + validate_disk_prealloc_against_compose(disk_prealloc, &request.compose_file)?; Ok(Manifest { id, @@ -333,9 +335,66 @@ pub fn create_manifest_from_vm_config( swtpm, networks, volumes, + disk_prealloc, }) } +/// Resolve the data disk preallocation for a deployment. This is a host +/// storage decision, so it comes from the request or the node default and +/// never from app-compose. +fn disk_prealloc_from_vm_config( + request: &VmConfiguration, + cvm_config: &CvmConfig, +) -> Result { + let Some(mode) = request + .disk_prealloc + .as_deref() + .map(str::trim) + .filter(|mode| !mode.is_empty()) + else { + return Ok(cvm_config.disk_prealloc); + }; + mode.parse() + .context("invalid disk_prealloc, expected off, metadata, falloc or full") +} + +/// Reserving data blocks and discarding them pull in opposite directions: the +/// host reserves the space up front, then the guest hands it straight back on +/// the first fstrim, leaving a disk that is neither reserved nor thin. Reject +/// the pair at deployment instead of letting the operator find out from `du` +/// later. +/// +/// `metadata` is deliberately not covered. It reserves no data blocks, so +/// discard costs it nothing, and rejecting it would force a compose change -- +/// a new compose hash, a new app id, another on-chain whitelist entry -- for a +/// combination that is not actually contradictory. +fn validate_disk_prealloc_against_compose( + prealloc: DiskPrealloc, + compose_file: &str, +) -> Result<()> { + if !prealloc.reserves_data_blocks() || !storage_discard_from_compose(compose_file)? { + return Ok(()); + } + bail!( + "disk preallocation ({}) reserves host blocks, so app-compose must set \ + storage_discard = false; either turn discard off in the compose file, or redeploy \ + with disk_prealloc = \"off\" or \"metadata\" -- the mode is fixed at deployment", + prealloc.as_str() + ) +} + +/// Read `storage_discard` out of app-compose. Everything else in the document +/// stays opaque, and a missing field means the app-compose default, which is +/// discard enabled. +fn storage_discard_from_compose(compose_file: &str) -> Result { + let compose: serde_json::Value = + serde_json::from_str(compose_file).context("invalid app compose JSON")?; + Ok(compose + .get("storage_discard") + .and_then(serde_json::Value::as_bool) + .unwrap_or(true)) +} + /// Extract only the field understood by this VMM. Keep every other app-compose /// field opaque so newer guest schemas and legacy third-party clients remain /// compatible with older VMMs. @@ -810,10 +869,26 @@ impl RpcHandler { if hda_path.exists() { info!("Resizing disk to {}GB", disk_size); let new_size_str = format!("{}G", disk_size); - let output = std::process::Command::new("qemu-img") - .args(["resize", &hda_path.display().to_string(), &new_size_str]) - .output() - .context("Failed to resize disk")?; + // Grow the disk the same way it was created, so a VM + // deployed with preallocation keeps its space reserved. + let mut args = vec!["resize".to_string()]; + if !manifest.disk_prealloc.is_off() { + args.push(format!( + "--preallocation={}", + manifest.disk_prealloc.as_str() + )); + } + args.push(hda_path.display().to_string()); + args.push(new_size_str); + // Off the async executor, for the same reason the launch + // path does it: growing a preallocated disk reserves the + // added space before it returns, and `full` writes it. + let output = tokio::task::spawn_blocking(move || { + std::process::Command::new("qemu-img").args(&args).output() + }) + .await + .context("disk resize task failed")? + .context("Failed to resize disk")?; if !output.status.success() { bail!( "Failed to resize disk: {}", @@ -849,27 +924,29 @@ impl VmmRpc for RpcHandler { warn!("Failed to set started: {}", err); } - let result = self + if let Err(err) = self .app .load_vm(&work_dir, &Default::default(), false) .await - .context("Failed to load VM"); - let result = match result { - Ok(()) => { - if !request.stopped { - self.app.start_vm(&id).await - } else { - Ok(()) - } - } - Err(err) => Err(err), - }; - if let Err(err) = result { + .context("Failed to load VM") + { + // Never started, so netd holds nothing for it. if let Err(err) = fs::remove_dir_all(&work_dir) { warn!("Failed to remove work dir: {}", err); } return Err(err); } + if !request.stopped { + if let Err(err) = self.app.start_vm(&id).await { + // A failed start may leave interfaces netd could not release. + // The normal removal keeps the directory, and with it + // `.netd-pending`, until they are gone. + if let Err(remove_err) = self.app.remove_vm(&id).await { + warn!(vm_id = %id, "failed to remove VM after start failure: {remove_err:#}"); + } + return Err(err); + } + } Ok(Id { id }) } @@ -928,6 +1005,21 @@ impl VmmRpc for RpcHandler { async fn update_vm(self, request: UpdateVmRequest) -> Result { info!(vm_id = %request.id, "update_vm RPC called"); + // A VM being removed is not one to reconfigure. Before the lock, + // because removal holds it across the whole teardown and anything that + // only asked afterwards would wait that out in order to be told no. + self.app.refuse_if_removing(&request.id)?; + // Held from here rather than around the parts that touch the host, + // because everything below writes into the workdir -- the compose + // file first, the manifest last -- and `put_manifest` creates the + // directory it writes into. An update that resumed after a removal + // deleted that directory would recreate it holding nothing but a + // manifest: invisible to `list_vms`, unloadable at every start, and + // claiming the VM's netd interfaces against collection forever. + let _launch = self.app.launch_lock(&request.id).await; + // Again under the lock: removal can have claimed the VM while this + // waited for it. + self.app.refuse_if_removing(&request.id)?; let new_id = if !request.compose_file.is_empty() { // check the compose file is valid let _app_compose: AppCompose = @@ -936,6 +1028,14 @@ impl VmmRpc for RpcHandler { if !compose_file_path.exists() { bail!("The instance {} not found", request.id); } + // Read the manifest here rather than reusing the one below, so a + // rejected update leaves the stored compose file untouched. + let manifest = self + .app + .work_dir(&request.id)? + .manifest() + .context("Failed to read manifest")?; + validate_disk_prealloc_against_compose(manifest.disk_prealloc, &request.compose_file)?; fs::write(compose_file_path, &request.compose_file) .context("Failed to write compose file")?; @@ -1008,18 +1108,19 @@ impl VmmRpc for RpcHandler { let networks = networks_from_proto(&request.networks, &cvm)?; resolve_requested_networks(&networks, &cvm, manifest.vcpu)? }; + // Under the launch lock this whole call holds. Reading "not + // running" outside it and acting on the answer inside is the exact + // race the lock exists to close: a launch can start, prepare its + // interfaces and deploy QEMU in between, and the release would + // then delete the interfaces of a VM that is running -- silently, + // since QEMU stays up and the supervisor still reports it healthy. let is_running = self .app .supervisor .info(&request.id) .await? .is_some_and(|info| info.state.status.is_running()); - if !is_running { - let runtime_networks = vm_work_dir.runtime_networks(); - self.app - .remove_netd_networks(&request.id, &runtime_networks) - .await - .context("failed to remove previous netd-managed networking")?; + if !is_running && self.app.release_vm_interfaces(&request.id).await { vm_work_dir.clear_runtime_networks()?; } manifest.networks = networks; @@ -1094,6 +1195,12 @@ impl VmmRpc for RpcHandler { "resize_vm RPC called" ); validate_resize_request(&request)?; + // The same guard as `update_vm`, for the same reason: this writes the + // manifest, and `put_manifest` recreates a directory a removal has + // just deleted. Before the lock, and again under it. + self.app.refuse_if_removing(&request.id)?; + let _launch = self.app.launch_lock(&request.id).await; + self.app.refuse_if_removing(&request.id)?; let vm_work_dir = self.app.work_dir(&request.id)?; let mut manifest = vm_work_dir.manifest().context("failed to read manifest")?; self.apply_resource_updates( @@ -1418,6 +1525,7 @@ mod tests { simulated_tee: None, networking: None, networks: vec![], + disk_prealloc: None, } } @@ -1490,6 +1598,114 @@ mod tests { assert!(manifest.networks.is_empty()); } + /// Preallocation is only accepted together with discard disabled, so a + /// preallocating request needs a compose file that says so. + fn test_vm_configuration_without_discard() -> VmConfiguration { + VmConfiguration { + compose_file: r#"{"storage_discard": false}"#.to_string(), + ..test_vm_configuration() + } + } + + #[test] + fn disk_prealloc_defaults_to_node_config_and_accepts_request_override() { + let mut cvm_config = test_cvm_config(); + assert_eq!(cvm_config.disk_prealloc, DiskPrealloc::Off); + + let manifest = + create_manifest_from_vm_config(test_vm_configuration(), &cvm_config).unwrap(); + assert_eq!(manifest.disk_prealloc, DiskPrealloc::Off); + + cvm_config.disk_prealloc = DiskPrealloc::Falloc; + let manifest = + create_manifest_from_vm_config(test_vm_configuration_without_discard(), &cvm_config) + .unwrap(); + assert_eq!(manifest.disk_prealloc, DiskPrealloc::Falloc); + + let mut request = test_vm_configuration_without_discard(); + request.disk_prealloc = Some("metadata".to_string()); + let manifest = create_manifest_from_vm_config(request, &cvm_config).unwrap(); + assert_eq!(manifest.disk_prealloc, DiskPrealloc::Metadata); + + // An empty string is the same as an unset field, as older clients and + // JSON round-trips produce it. + let mut request = test_vm_configuration_without_discard(); + request.disk_prealloc = Some(String::new()); + let manifest = create_manifest_from_vm_config(request, &cvm_config).unwrap(); + assert_eq!(manifest.disk_prealloc, DiskPrealloc::Falloc); + + // Opting one VM out of a preallocating node default needs no compose + // change, so the app identity stays put. + let mut request = test_vm_configuration(); + request.disk_prealloc = Some("off".to_string()); + let manifest = create_manifest_from_vm_config(request, &cvm_config).unwrap(); + assert_eq!(manifest.disk_prealloc, DiskPrealloc::Off); + } + + #[test] + fn create_rejects_unknown_disk_prealloc_mode() { + let mut request = test_vm_configuration(); + request.disk_prealloc = Some("sparse".to_string()); + let err = create_manifest_from_vm_config(request, &test_cvm_config()).unwrap_err(); + assert!(err.to_string().contains("disk_prealloc"), "{err}"); + } + + #[test] + fn preallocation_is_rejected_while_the_guest_may_discard() { + let cvm_config = test_cvm_config(); + + // storage_discard defaults to true in app-compose, so an unset field + // conflicts just as an explicit `true` does. + for compose in ["{}", r#"{"storage_discard": true}"#] { + let request = VmConfiguration { + compose_file: compose.to_string(), + disk_prealloc: Some("falloc".to_string()), + ..test_vm_configuration() + }; + let err = create_manifest_from_vm_config(request, &cvm_config).unwrap_err(); + assert!( + err.to_string().contains("storage_discard"), + "{compose}: {err}" + ); + } + + // The same conflict, reached through the node default instead of the + // request. + let preallocating_node = CvmConfig { + disk_prealloc: DiskPrealloc::Full, + ..cvm_config + }; + let err = create_manifest_from_vm_config(test_vm_configuration(), &preallocating_node) + .unwrap_err(); + assert!(err.to_string().contains("storage_discard"), "{err}"); + } + + #[test] + fn metadata_preallocation_coexists_with_discard() { + // Rejecting it would cost a compose change -- and with it a new app id + // -- to avoid a conflict that does not exist: no data block is + // reserved, so there is none for the guest to hand back. + let mut request = test_vm_configuration(); + request.disk_prealloc = Some("metadata".to_string()); + let manifest = create_manifest_from_vm_config(request, &test_cvm_config()).unwrap(); + assert_eq!(manifest.disk_prealloc, DiskPrealloc::Metadata); + } + + #[test] + fn a_malformed_compose_is_reported_as_such() { + // Not reachable today (the callers parse the compose first), but the + // discard lookup must not turn "broken JSON" into "turn discard off". + let mut request = test_vm_configuration(); + request.compose_file = "{not json".to_string(); + request.disk_prealloc = Some("falloc".to_string()); + let err = format!( + "{:#}", + create_manifest_from_vm_config(request, &test_cvm_config()).unwrap_err() + ); + assert!(err.contains("compose"), "{err}"); + assert!(!err.contains("storage_discard"), "{err}"); + } + #[test] fn resize_request_rejects_empty_zero_and_empty_image_updates() { let mut request = ResizeVmRequest { @@ -1772,6 +1988,9 @@ mod tests { .expect("a named backend is an override"); } + /// Deliberately restated rather than calling `NetworkingMode::as_str`: the + /// test below checks that what `GetInfo` reports is accepted back, and a + /// helper that shares the production mapping could only ever agree with it. fn networking_mode_name_for_test(mode: NetworkingMode) -> &'static str { match mode { NetworkingMode::Bridge => "bridge", diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index e13facc73..f132f5bda 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -9,6 +9,7 @@ //! request and response messages. use std::{ + collections::HashSet, fs::{File, OpenOptions, Permissions}, io::{self, Write as _}, os::{ @@ -31,14 +32,15 @@ use ra_rpc::{CallContext, RpcCall}; use rocket::listener::{unix::UnixStream as RocketUnixStream, Endpoint, Listener}; use sha2::{Digest, Sha256}; use tokio::net::UnixListener; -use tracing::{info, warn}; +use tracing::{debug, info, warn}; use uuid::Uuid; use wait_timeout::ChildExt; use dstack_netd_rpc::netd_server::{NetdRpc, NetdServer}; pub use dstack_netd_rpc::{ - netd_client::NetdClient, CheckInterfaceRequest, InterfaceIdentity, PrepareBridgeRequest, - PrepareMacvtapRequest, PreparedInterface, + netd_client::NetdClient, CheckInterfaceRequest, InterfaceIdentity, InterfaceList, + InterfaceName, InterfaceRecord, ListInterfacesRequest, PrepareBridgeRequest, + PrepareMacvtapRequest, PreparedInterface, RemoveVmResponse, VmRef, }; use crate::config::{NetdConfig, NetworkFilterConfig}; @@ -47,12 +49,28 @@ use crate::config::{NetdConfig, NetworkFilterConfig}; /// operations queued ahead of it. const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); const COMMAND_TIMEOUT: Duration = Duration::from_secs(30); +/// Bound on listing nwfilter bindings. See [`existing_bindings`]. +const LISTING_TIMEOUT: Duration = Duration::from_secs(10); const IP_PATH: &str = "/usr/sbin/ip"; const VIRSH_PATH: &str = "/usr/bin/virsh"; const LOCK_PATH: &str = "/run/lock/dstack-netd.lock"; /// Upper bound on TAP queue pairs netd will create. Mirrors the VMM's own cap /// so a malformed request cannot ask the kernel for an unbounded device. const MAX_QUEUES: u32 = 64; +/// Highest NIC index an identity may name. Also the width of the space a +/// whole-VM sweep has to enumerate, since it derives names instead of reading a +/// record. +const MAX_NIC_INDEX: u32 = 255; +/// The interface names netd may create. Reserved: anything matching it is +/// netd's to delete, and nothing else on the host may take one. +const TAP_PREFIX: &str = "dt"; +/// Hex characters of digest in a TAP name, after [`TAP_PREFIX`]. +const TAP_DIGEST_CHARS: usize = 12; +/// Version tag on the ownership record. Present so a later format can be told +/// from this one rather than mis-parsed as it. +const ALIAS_PREFIX: &str = "dstack1"; +/// What the kernel stores in an interface alias, minus the terminator. +const MAX_IFALIAS: usize = 255; pub fn tap_name(identity: &InterfaceIdentity) -> String { let input = format!( @@ -60,7 +78,82 @@ pub fn tap_name(identity: &InterfaceIdentity) -> String { identity.instance_id, identity.vm_id, identity.nic_index ); let digest = Sha256::digest(input.as_bytes()); - format!("dt{}", hex::encode(&digest[..6])) + format!( + "{TAP_PREFIX}{}", + hex::encode(&digest[..TAP_DIGEST_CHARS / 2]) + ) +} + +/// The ownership record netd writes onto every interface it creates. +/// +/// The record lives on the resource, so it has exactly the resource's +/// lifetime. A file under `/run` would be a second thing to keep in step with +/// the first, and the failure this whole path exists to fix is precisely a +/// record that got out of step: written after the interface, lost with the +/// directory, and unreadable to anything but the process that wrote it. +/// +/// Never trusted as *authority*. Anything that can reach this socket can also +/// name an identity, and the interface name is a digest of that identity -- +/// so a record is believed only when re-deriving the name from it reproduces +/// the name it is written on. Ambiguity (a separator inside an identity), +/// truncation, and forgery all fail that check and land in the same bucket as +/// no record at all, which is the bucket handled conservatively. +pub fn interface_alias(identity: &InterfaceIdentity) -> String { + format!( + "{ALIAS_PREFIX}:{}:{}:{}", + identity.nic_index, identity.instance_id, identity.vm_id + ) +} + +/// The identity an interface claims, if the claim checks out. +/// +/// `nic_index` first, so the two free-form fields are the last two and a +/// `vm_id` containing the separator still parses. An `instance_id` containing +/// one does not, and is refused at prepare rather than mis-parsed here. +pub fn owner_of(tap: &str, alias: &str) -> Option { + // `trim_end_matches`, not `trim`: sysfs adds a newline, and a `vm_id` + // whose own trailing whitespace were trimmed off here would re-derive a + // name that is not the one it is on, making the interface permanently + // unattributable -- never collected, only removable by hand. + let rest = alias + .trim_end_matches(['\n', '\r']) + .strip_prefix(ALIAS_PREFIX)? + .strip_prefix(':')?; + let (nic_index, rest) = rest.split_once(':')?; + let (instance_id, vm_id) = rest.split_once(':')?; + let identity = InterfaceIdentity { + instance_id: instance_id.to_string(), + vm_id: vm_id.to_string(), + nic_index: nic_index.parse().ok()?, + }; + // The name is the proof. A record that does not reproduce it describes + // some other interface, or nothing. + (tap_name(&identity) == tap).then_some(identity) +} + +/// Whether this name is one netd can have created. See [`TAP_PREFIX`]. +pub fn is_managed_name(interface: &str) -> bool { + let Some(digest) = interface.strip_prefix(TAP_PREFIX) else { + return false; + }; + digest.len() == TAP_DIGEST_CHARS + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// Rejects an instance ID no interface could be recorded as belonging to. +/// +/// At startup rather than at the first launch. The VMM derives one that is +/// always valid; an operator who configured their own learns here rather than +/// from the first VM that fails to get a NIC. +pub fn validate_instance_id(instance_id: &str) -> Result<()> { + validate_identity(&InterfaceIdentity { + instance_id: instance_id.to_string(), + vm_id: "0".repeat(64), + nic_index: MAX_NIC_INDEX, + }) + .context("invalid cvm.instance_id") } pub fn instance_id(configured: &str, run_path: &Path) -> String { @@ -278,6 +371,39 @@ impl NetdRpc for NetdHandler { .await } + async fn remove_vm(self, request: VmRef) -> Result { + self.run(move |config| { + let removed = sweep_vm_interfaces( + &config.libvirt_uri, + &request.instance_id, + &request.vm_id, + config.filter_policy().requires_binding(), + )?; + Ok(RemoveVmResponse { removed }) + }) + .await + } + + async fn remove_interface_by_name(self, request: InterfaceName) -> Result<()> { + self.run(move |config| { + remove_interface_by_name( + &config.libvirt_uri, + &request.tap, + config.filter_policy().requires_binding(), + ) + }) + .await + } + + async fn list_interfaces(self, request: ListInterfacesRequest) -> Result { + self.run(move |config| { + Ok(InterfaceList { + interfaces: list_interfaces(&config.libvirt_uri, &request.instance_id), + }) + }) + .await + } + async fn check_interface(self, request: CheckInterfaceRequest) -> Result<()> { self.run(move |config| { let identity = required_identity(&request.identity)?; @@ -361,6 +487,7 @@ fn prepare_macvtap( add.extend_from_slice(&["type", "macvtap", "mode", mode]); ip(&add)?; let result = (|| { + set_alias(&tap, identity)?; let ifindex = std::fs::read_to_string(Path::new("/sys/class/net").join(&tap).join("ifindex")) .context("failed to read macvtap ifindex")?; @@ -384,7 +511,7 @@ fn prepare_macvtap( })(); match result { Ok(device) => { - info!(%tap, %parent, %mode, %device, %queues, "prepared macvtap"); + info!(%tap, %parent, %mode, %device, %queues, workdir = %request.workdir, "prepared macvtap"); Ok(PreparedInterface { tap, device, @@ -461,6 +588,10 @@ fn prepare_bridge( add.extend_from_slice(&["user", &uid]); ip(&add)?; let result = (|| { + // Before anything else it could fail at. An interface that exists + // without a record is one nothing can attribute, and the window in + // which that is true is the window a crash turns permanent. + set_alias(&tap, identity)?; ip(&["link", "set", "dev", &tap, "master", &request.bridge])?; if filtered { let xml = binding_xml(identity, &request.mac, &tap, filter); @@ -477,7 +608,14 @@ fn prepare_bridge( let _ = remove_interface(libvirt_uri, &tap, BindingCleanup::BestEffort); return Err(error); } - info!(%tap, bridge = %request.bridge, %filtered, %queues, "prepared TAP"); + info!( + %tap, + bridge = %request.bridge, + %filtered, + %queues, + workdir = %request.workdir, + "prepared TAP" + ); Ok(PreparedInterface { tap, device: String::new(), @@ -497,11 +635,254 @@ enum BindingCleanup { /// be running at all, and a stale binding left by an earlier, filtered /// interface at this name is still worth clearing when it is. BestEffort, + /// The caller has already decided about the binding. Used by a pass over + /// many interfaces, which asks libvirt once about all of them rather than + /// once per interface. + Skip, +} + +/// Deletes every interface a VM could hold, by deriving each name rather than +/// consulting a record. +/// +/// `validate_identity` caps the NIC index, so the whole space a VM can occupy +/// is enumerable: 256 names, each a `stat` that usually misses. Cleanup is +/// best-effort about unknown bindings on unfiltered nodes, which need not have +/// libvirtd at all. Filtered nodes must confirm their bindings were released. +/// +/// A name with no interface is not skipped. An nwfilter binding outlives the +/// TAP it was bound to, so the one state teardown must not leave behind is +/// exactly the one a `/sys/class/net` check cannot see: the per-name Remove +/// this replaced deleted the binding unconditionally, and a sweep that reaches +/// less than the thing it replaced is not a sweep. Those names are decided +/// against a single listing, because the whole point of enumerating a bounded +/// space is that deciding one name stays cheap. +fn sweep_vm_interfaces( + libvirt_uri: &str, + instance_id: &str, + vm_id: &str, + requires_binding: bool, +) -> Result { + let identity = InterfaceIdentity { + instance_id: instance_id.to_string(), + vm_id: vm_id.to_string(), + nic_index: 0, + }; + validate_identity(&identity)?; + let listing = existing_bindings(libvirt_uri); + // Not `listing.is_ok()`. A listing that could not be produced says + // nothing about whether a *deletion* will work, and reading it as "libvirt + // is down, skip the bindings" would mean a node whose listing breaks for + // any reason silently stops cleaning up bindings at all -- which is worse + // than the per-name asking this listing exists to avoid. The pass finds + // out by trying, once. The exception is a listing that timed out: a + // `libvirtd` that did not answer it will not answer a deletion either, and + // waiting for that one too would outlast the caller's request. + let mut libvirt = !listing.as_ref().is_err_and(timed_out); + let bindings = listing.ok(); + let mut removed = 0; + let mut first_error = (requires_binding && bindings.is_none()) + .then(|| anyhow::anyhow!("cannot confirm nwfilter cleanup: binding listing failed")); + for nic_index in 0..=MAX_NIC_INDEX { + let tap = tap_name(&InterfaceIdentity { + nic_index, + ..identity.clone() + }); + let present = Path::new("/sys/class/net").join(&tap).exists(); + // A pass gets one answer about libvirt, not one per interface. Asking + // again after it has failed is how a hung `libvirtd` turns a bounded + // collection into an unbounded one. + let known_binding = bindings.as_ref().is_some_and(|held| held.contains(&tap)); + if present { + take_down(&tap); + } + let macvtap = present && is_macvtap(&tap); + // A device netd did not create is refused below, before anything at + // its name is touched, as a manual removal refuses it. + let foreign = present && !macvtap && !is_tuntap(&tap); + // A macvtap never carries a binding of its own, but a name that was a + // filtered TAP before can still hold one that is listed. + let wanted = !foreign && (known_binding || (present && !macvtap)); + if libvirt && wanted { + if let Err(error) = delete_binding(libvirt_uri, &tap) { + warn!(%tap, %error, "failed to remove an nwfilter binding"); + libvirt = false; + // Unfiltered nodes do not require a running libvirtd. An + // unknown, possible binding must not make a successful TAP + // deletion fail there. Still retain failures for bindings we + // actually found, including ones left by an older policy. + if requires_binding || known_binding { + first_error.get_or_insert(error); + } + } else if !present { + info!(%tap, %vm_id, "removed orphaned nwfilter binding"); + } + } + if !libvirt && known_binding { + first_error + .get_or_insert_with(|| anyhow::anyhow!("nwfilter binding {tap} was not released")); + } + if !present { + continue; + } + // Keep going after a failure. Stopping at the first one would leave the + // rest of a VM's interfaces behind over one that is stuck. + match remove_interface(libvirt_uri, &tap, BindingCleanup::Skip) { + Err(error) => { + warn!(%tap, %error, "failed to remove interface"); + first_error.get_or_insert(error); + } + Ok(()) => { + info!(%tap, %vm_id, "removed interface"); + removed += 1; + } + } + } + match first_error { + Some(error) => Err(error).context("failed to remove every interface for this VM"), + None => Ok(removed), + } +} + +/// Deletes one interface, and any binding at its name, for an operator. +/// +/// `netd list` shows a binding whose interface is gone as its own row, and this +/// is how an operator removes it, so a binding that is known to exist, or that +/// is all the name could still hold, has to be confirmed gone before this +/// reports success. Only a binding that merely might sit next to a live +/// interface on a node that does not filter is best effort: `libvirtd` need +/// not be running there, and the interface still has to go. The same rules as +/// [`sweep_vm_interfaces`]. +fn remove_interface_by_name(libvirt_uri: &str, tap: &str, requires_binding: bool) -> Result<()> { + if !is_managed_name(tap) { + bail!("{tap} is not a name netd could have created"); + } + let present = Path::new("/sys/class/net").join(tap).exists(); + let macvtap = present && is_macvtap(tap); + if present && !macvtap && !is_tuntap(tap) { + bail!("refusing to delete {tap}: it is neither a tun/tap nor a macvtap device"); + } + if present { + take_down(tap); + } + let possible = |error: anyhow::Error| { + warn!(%tap, "could not clear a possible nwfilter binding: {error:#}"); + None + }; + let binding_error = match existing_bindings(libvirt_uri).map(|held| held.contains(tap)) { + // Nothing listed at the name, and nothing there that could carry one. + Ok(false) if !present || macvtap => None, + Ok(known) => match delete_binding(libvirt_uri, tap) { + Ok(()) => None, + Err(error) if known || requires_binding => Some(error), + Err(error) => possible(error), + }, + // A macvtap carries no binding of its own, and there is no listing + // saying an older one is left. A filtering node still needs that + // confirmed, as a sweep does. + Err(_) if macvtap && !requires_binding => None, + Err(listing_error) => { + // A `libvirtd` that did not answer the listing will not answer a + // deletion either, and waiting for it would outlast the request. + let attempt = if timed_out(&listing_error) { + Err(listing_error.context("cannot confirm the binding is gone")) + } else { + delete_binding(libvirt_uri, tap).map_err(|error| { + error.context(format!( + "cannot confirm the binding is gone: {listing_error:#}" + )) + }) + }; + match attempt { + Ok(()) => None, + Err(error) if present && !requires_binding => possible(error), + Err(error) => Some(error), + } + } + }; + // The interface goes even when its binding could not: a TAP left on the + // bridge is worse than a binding with nothing to filter. + remove_interface(libvirt_uri, tap, BindingCleanup::Skip)?; + match binding_error { + Some(error) => Err(error), + None => Ok(()), + } +} + +/// Every resource netd owns, read off the host rather than out of a record. +/// +/// Ownership is the reserved name plus the kernel's own answer about what kind +/// of device it is; attribution is the interface's alias, checked by +/// re-deriving the name from it. A listing never fails for want of libvirt: on +/// a node that does not filter, `libvirtd` need not be running, and an +/// interface inventory that refused to be produced without it would be +/// unavailable exactly where unfiltered TAPs live. +fn list_interfaces(libvirt_uri: &str, instance_id: &str) -> Vec { + let bindings = existing_bindings(libvirt_uri).ok(); + let mut records = Vec::new(); + let mut seen = HashSet::new(); + if let Ok(entries) = std::fs::read_dir("/sys/class/net") { + for entry in entries.flatten() { + let Ok(tap) = entry.file_name().into_string() else { + continue; + }; + if !is_managed_name(&tap) { + continue; + } + let kind = if is_macvtap(&tap) { + "macvtap" + } else if is_tuntap(&tap) { + "tap" + } else { + // The name is netd's to use, but this is not a device netd + // creates. Listing it would invite a caller to delete it. + continue; + }; + let alias = + std::fs::read_to_string(Path::new("/sys/class/net").join(&tap).join("ifalias")) + .unwrap_or_default(); + let owner = owner_of(&tap, &alias); + seen.insert(tap.clone()); + records.push(InterfaceRecord { + kind: kind.to_string(), + nic_index: owner.as_ref().map(|identity| identity.nic_index), + instance_id: owner.as_ref().map(|identity| identity.instance_id.clone()), + vm_id: owner.map(|identity| identity.vm_id), + tap, + }); + } + } + // A binding outlives its interface, and an interface is the only thing that + // carries a record, so an orphaned binding can never be attributed. It is + // still netd's: nothing else creates a binding at one of these names. + for name in bindings.into_iter().flatten() { + if !seen.contains(&name) { + records.push(InterfaceRecord { + tap: name, + kind: "binding".to_string(), + instance_id: None, + vm_id: None, + nic_index: None, + }); + } + } + if !instance_id.is_empty() { + records.retain(|record| record.instance_id.as_deref() == Some(instance_id)); + } + records.sort_by(|left, right| left.tap.cmp(&right.tap)); + records } fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Result<()> { let macvtap = is_macvtap(tap); if Path::new("/sys/class/net").join(tap).exists() { + // The name is 48 bits of SHA-256, so a collision is not the worry. A + // caller asserting an identity that happens to derive to some + // pre-existing device is: netd runs as root and `ip link delete` does + // not ask what it is deleting. netd creates exactly two kinds of + // device, and the kernel publishes an attribute unique to each. + if !macvtap && !is_tuntap(tap) { + bail!("refusing to delete {tap}: it is neither a tun/tap nor a macvtap device"); + } let _ = ip(&["link", "set", "dev", tap, "down"]); } // A macvtap interface never carries a binding. Anything else might: this @@ -509,6 +890,7 @@ fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Re // outlives the interface. if !macvtap { match cleanup { + BindingCleanup::Skip => {} BindingCleanup::Required => delete_binding(libvirt_uri, tap)?, BindingCleanup::BestEffort => { // netd refuses to start without virsh, so the binary is always @@ -527,6 +909,35 @@ fn remove_interface(libvirt_uri: &str, tap: &str, cleanup: BindingCleanup) -> Re Ok(()) } +/// Sets a device netd created down before anything else about it is torn down. +/// +/// A stop only signals QEMU, so it may still be sending when its release runs. +/// Down first, so its traffic does not reach the bridge unfiltered for as long +/// as deleting the binding takes. Never touches a device netd did not create. +fn take_down(interface: &str) { + if is_macvtap(interface) || is_tuntap(interface) { + let _ = ip(&["link", "set", "dev", interface, "down"]); + } +} + +/// Records who an interface belongs to, on the interface. See +/// [`interface_alias`]. +fn set_alias(tap: &str, identity: &InterfaceIdentity) -> Result<()> { + let alias = interface_alias(identity); + ip(&["link", "set", "dev", tap, "alias", &alias]) + .with_context(|| format!("failed to record ownership on {tap}")) +} + +/// Whether this is a tun/tap device. `tun_flags` is published by the tun +/// driver and by nothing else, so its presence is the kernel's own answer -- +/// as `macvtap/` is for the other kind of device netd creates. +fn is_tuntap(interface: &str) -> bool { + Path::new("/sys/class/net") + .join(interface) + .join("tun_flags") + .exists() +} + fn is_macvtap(interface: &str) -> bool { Path::new("/sys/class/net") .join(interface) @@ -642,9 +1053,25 @@ fn validate_identity(identity: &InterfaceIdentity) -> Result<()> { bail!("invalid {label}"); } } - if identity.nic_index > 255 { + if identity.nic_index > MAX_NIC_INDEX { bail!("NIC index is out of range"); } + // An identity that cannot be recorded on the interface is refused rather + // than built unattributed. A host resource nothing can name the owner of + // is the thing this whole path exists to stop producing, and the kernel's + // alias is the only place with the interface's exact lifetime to put it. + let alias = interface_alias(identity); + if alias.len() > MAX_IFALIAS { + bail!( + "identity is too long to record on the interface: {} bytes of {MAX_IFALIAS}", + alias.len() + ); + } + // The record puts the two free-form fields last, so only the first of them + // has to be unambiguous. + if identity.instance_id.contains(':') { + bail!("instance ID must not contain ':'"); + } Ok(()) } @@ -705,13 +1132,71 @@ fn ip(args: &[&str]) -> Result<()> { } fn virsh(uri: &str, args: &[&str], stdin: Option<&[u8]>) -> Result<()> { + virsh_output(uri, args, stdin).map(|_| ()) +} + +fn virsh_output(uri: &str, args: &[&str], stdin: Option<&[u8]>) -> Result { let mut full_args = vec!["--connect", uri]; full_args.extend_from_slice(args); - run_command(VIRSH_PATH, &full_args, stdin) + run_command_with_timeout(VIRSH_PATH, &full_args, stdin, COMMAND_TIMEOUT) +} + +/// Every nwfilter binding libvirt holds at a name netd could have created. +/// +/// One call, so that a sweep can decide 256 names against a set instead of +/// asking libvirt 256 times. `None` means libvirt could not be asked at all, +/// which on a node running unfiltered TAPs is the normal state -- `virsh` must +/// be installed for netd to start, but `libvirtd` need not be running. +/// +/// The command has no machine-readable mode: it prints a two-line header and +/// then one binding per line, interface name first, and it accepts no options +/// at all -- `--name` is not one of them, and asking for it fails the whole +/// call. Narrowing to netd's own name space is what makes parsing a human +/// table safe: a header, a rule line, or a column that moves cannot produce a +/// `dt` name, and a binding at any other name is not netd's to reason about. +fn existing_bindings(uri: &str) -> Result> { + // Shorter than a mutation's timeout. A sweep or a manual removal asks this + // first, and the whole request has to fit in the VMM's own request + // timeout, which a hung `libvirtd` would otherwise use up on its own. + let output = run_command_with_timeout( + VIRSH_PATH, + &["--connect", uri, "nwfilter-binding-list"], + None, + LISTING_TIMEOUT, + ) + .inspect_err(|error| debug!("could not list nwfilter bindings: {error:#}"))?; + Ok(output + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|name| is_managed_name(name)) + .map(str::to_string) + .collect()) } fn run_command(program: &str, args: &[&str], stdin: Option<&[u8]>) -> Result<()> { - run_command_with_timeout(program, args, stdin, COMMAND_TIMEOUT) + run_command_with_timeout(program, args, stdin, COMMAND_TIMEOUT).map(|_| ()) +} + +/// A helper that did not finish in time. Typed so a caller can tell a hung +/// daemon, which will not answer the next request either, from a refusal. +#[derive(Debug)] +struct CommandTimedOut { + program: String, + timeout: Duration, +} + +impl std::fmt::Display for CommandTimedOut { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} timed out after {:?}", self.program, self.timeout) + } +} + +impl std::error::Error for CommandTimedOut {} + +fn timed_out(error: &anyhow::Error) -> bool { + error + .chain() + .any(|cause| cause.downcast_ref::().is_some()) } fn run_command_with_timeout( @@ -719,7 +1204,7 @@ fn run_command_with_timeout( args: &[&str], stdin: Option<&[u8]>, command_timeout: Duration, -) -> Result<()> { +) -> Result { let mut child = Command::new(program) .args(args) .stdin(if stdin.is_some() { @@ -731,27 +1216,47 @@ fn run_command_with_timeout( .stderr(Stdio::piped()) .spawn() .with_context(|| format!("failed to execute {program}"))?; + // Drain both pipes while waiting. A child whose output fills a pipe + // blocks until someone reads it, so waiting first and reading afterwards + // turns any output larger than the pipe buffer -- a binding listing on a + // busy host -- into a timeout. + let stdout = drain(child.stdout.take().context("missing command stdout")?); + let stderr = drain(child.stderr.take().context("missing command stderr")?); if let Some(input) = stdin { - child - .stdin - .take() - .context("missing command stdin")? - .write_all(input)?; + let mut pipe = child.stdin.take().context("missing command stdin")?; + let input = input.to_vec(); + // On its own thread for the same reason: a child that writes before + // it has read all of its input must not stall the write past the + // timeout. Dropping the pipe afterwards is the child's EOF. + std::thread::spawn(move || { + let _ = pipe.write_all(&input); + }); } - if child.wait_timeout(command_timeout)?.is_none() { + let Some(status) = child.wait_timeout(command_timeout)? else { let _ = child.kill(); let _ = child.wait(); - bail!( - "{} timed out after {command_timeout:?}", - Path::new(program).display() - ); - } - let output = child.wait_with_output()?; - if !output.status.success() { - let error = String::from_utf8_lossy(&output.stderr); + return Err(CommandTimedOut { + program: Path::new(program).display().to_string(), + timeout: command_timeout, + } + .into()); + }; + let stdout = stdout.join().unwrap_or_default(); + let stderr = stderr.join().unwrap_or_default(); + if !status.success() { + let error = String::from_utf8_lossy(&stderr); bail!("{} failed: {}", Path::new(program).display(), error.trim()); } - Ok(()) + Ok(String::from_utf8_lossy(&stdout).into_owned()) +} + +/// Reads a pipe to its end on its own thread. +fn drain(mut pipe: impl io::Read + Send + 'static) -> std::thread::JoinHandle> { + std::thread::spawn(move || { + let mut buffer = Vec::new(); + let _ = pipe.read_to_end(&mut buffer); + buffer + }) } fn require_executable(path: &str) -> Result<()> { @@ -775,6 +1280,176 @@ fn prepare_socket_path(socket: &Path) -> Result<()> { Ok(()) } +/// A netd that exists only to be talked to. +/// +/// The VMM's side of the netd conversation -- what it refuses, what it does +/// when netd refuses -- needs no privileged daemon, only something that serves +/// the same RPC on a socket. This is that, scripted per method, recording what +/// it was asked so a test can assert on the conversation rather than on its +/// effects. +#[cfg(test)] +pub(crate) mod testing { + use std::{ + path::{Path, PathBuf}, + sync::{Arc, Mutex}, + }; + + use anyhow::{bail, Result}; + use ra_rpc::{CallContext, RpcCall}; + use serde_json::Value; + use tokio::net::UnixListener; + + use super::{ + CheckInterfaceRequest, InterfaceIdentity, InterfaceList, InterfaceName, + ListInterfacesRequest, NetdListener, NetdRpc, NetdServer, PrepareBridgeRequest, + PrepareMacvtapRequest, PreparedInterface, RemoveVmResponse, VmRef, + }; + + #[derive(Clone)] + pub(crate) struct FakeState { + /// Methods answered with success; every other one is refused. + handles: Arc>, + seen: Arc>>, + } + + pub(crate) struct FakeHandler(FakeState); + + impl RpcCall for FakeHandler { + type PrpcService = NetdServer; + + fn construct(context: CallContext<'_, FakeState>) -> Result { + Ok(Self(context.state.clone())) + } + } + + impl FakeHandler { + fn record(&self, method: &str, request: impl serde::Serialize) -> Result<()> { + self.0 + .seen + .lock() + .expect("poisoned") + .push((method.to_string(), serde_json::to_value(request)?)); + if !self.0.handles.iter().any(|name| name == method) { + bail!("fake netd refuses {method}"); + } + Ok(()) + } + } + + impl NetdRpc for FakeHandler { + async fn prepare_bridge(self, request: PrepareBridgeRequest) -> Result { + let queues = request.queues.max(1); + self.record("PrepareBridge", request)?; + Ok(PreparedInterface { + tap: "dtdeadbeef00".into(), + device: String::new(), + queues, + }) + } + + async fn prepare_macvtap( + self, + request: PrepareMacvtapRequest, + ) -> Result { + let queues = request.queues.max(1); + self.record("PrepareMacvtap", request)?; + Ok(PreparedInterface { + tap: "dtdeadbeef00".into(), + device: "/dev/tap1".into(), + queues, + }) + } + + async fn remove_interface(self, request: InterfaceIdentity) -> Result<()> { + self.record("RemoveInterface", request) + } + + async fn remove_vm(self, request: VmRef) -> Result { + self.record("RemoveVm", request)?; + Ok(RemoveVmResponse { removed: 0 }) + } + + async fn remove_interface_by_name(self, request: InterfaceName) -> Result<()> { + self.record("RemoveInterfaceByName", request) + } + + async fn list_interfaces(self, request: ListInterfacesRequest) -> Result { + self.record("ListInterfaces", request)?; + Ok(InterfaceList::default()) + } + + async fn check_interface(self, request: CheckInterfaceRequest) -> Result<()> { + self.record("CheckInterface", request) + } + } + + pub(crate) struct FakeNetd { + _dir: Option, + socket: PathBuf, + seen: Arc>>, + } + + impl FakeNetd { + /// Serves on a fresh socket, answering `handles` and refusing the + /// rest. The socket is bound before this returns, so a caller can + /// connect at once. + pub(crate) fn spawn(handles: &[&str]) -> Self { + let dir = tempfile::tempdir().expect("tempdir"); + let mut netd = Self::spawn_at(&dir.path().join("netd.sock"), handles); + netd._dir = Some(dir); + netd + } + + /// As [`FakeNetd::spawn`], on a socket path the caller chose -- for a + /// netd that comes up after the VMM already started asking for it. + pub(crate) fn spawn_at(socket: &Path, handles: &[&str]) -> Self { + let socket = socket.to_path_buf(); + let listener = UnixListener::bind(&socket).expect("bind"); + let state = FakeState { + handles: Arc::new(handles.iter().map(|name| name.to_string()).collect()), + seen: Arc::new(Mutex::new(Vec::new())), + }; + let seen = state.seen.clone(); + tokio::spawn(async move { + let figment = rocket::Config::figment().merge(("log_level", "off")); + let ignite = rocket::custom(figment) + .manage(state) + .mount("/prpc", ra_rpc::prpc_routes!(FakeState, FakeHandler)) + .ignite() + .await + .expect("ignite fake netd"); + let _ = ignite.launch_on(NetdListener(listener)).await; + }); + Self { + _dir: None, + socket, + seen, + } + } + + pub(crate) fn socket(&self) -> &Path { + &self.socket + } + + /// Every request it was sent, in order. + pub(crate) fn seen(&self) -> Vec { + self.calls() + .into_iter() + .map(|(_, request)| request) + .collect() + } + + /// The method of every request it was sent, in order. + pub(crate) fn methods(&self) -> Vec { + self.calls().into_iter().map(|(method, _)| method).collect() + } + + fn calls(&self) -> Vec<(String, Value)> { + self.seen.lock().expect("poisoned").clone() + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -806,6 +1481,7 @@ mod tests { qemu_uid: 1000, filtered: true, queues: 0, + workdir: String::new(), }; let filter = NetworkFilterConfig { mode: crate::config::NetworkFilterMode::Libvirt, @@ -857,6 +1533,7 @@ mod tests { qemu_uid: 1000, filtered: false, queues: 4, + workdir: String::new(), }; let filtering = NetworkFilterConfig { mode: crate::config::NetworkFilterMode::Libvirt, @@ -907,6 +1584,7 @@ mod tests { qemu_uid: 1000, mode: "bridge".into(), queues: 4, + workdir: String::new(), }; let error = match prepare_macvtap("test:///default", &request, &filtering) { Err(error) => error, @@ -928,6 +1606,7 @@ mod tests { qemu_uid: 1000, filtered: true, queues: 1, + workdir: String::new(), }; // Nothing on the wire can name a filter: the field does not exist. let wire = serde_json::to_value(&request).unwrap(); @@ -961,6 +1640,325 @@ mod tests { .unwrap_err(); assert!(error.to_string().contains("timed out")); assert!(started.elapsed() < Duration::from_secs(2)); + // Output larger than a pipe buffer is read, not mistaken for a hang. + let large = run_command_with_timeout( + "/bin/sh", + &["-c", "head -c 1000000 /dev/zero | tr '\\0' x"], + None, + Duration::from_secs(10), + ) + .unwrap(); + assert_eq!(large.len(), 1_000_000); + // Input is still delivered, and its EOF still arrives. + let echoed = + run_command_with_timeout("/bin/cat", &[], Some(b"binding xml"), COMMAND_TIMEOUT) + .unwrap(); + assert_eq!(echoed, "binding xml"); + // Recognised through whatever context a caller adds. + assert!(timed_out(&error.context("while listing"))); + let refused = run_command_with_timeout("/bin/sh", &["-c", "exit 1"], None, COMMAND_TIMEOUT) + .unwrap_err(); + assert!(!timed_out(&refused)); + } + + /// A whole-VM sweep names a VM and no NIC: reaching the indices the caller + /// can no longer name is the point, so netd derives the whole space. That + /// space is bounded by what an identity may say, which is what makes + /// deriving it cheap enough to do on every launch. + #[test] + fn a_whole_vm_sweep_covers_a_bounded_space() { + let value = serde_json::to_value(VmRef { + instance_id: "instance".into(), + vm_id: "vm".into(), + }) + .unwrap(); + assert!(value.get("nic_index").is_none()); + + let mut identity = identity("instance", "vm", MAX_NIC_INDEX); + assert!(validate_identity(&identity).is_ok()); + identity.nic_index = MAX_NIC_INDEX + 1; + assert!(validate_identity(&identity).is_err()); + } + + /// The record is a hint; the name is the proof. Everything that can go + /// wrong with reading a string off an interface -- forged, truncated, + /// ambiguous, absent -- has to land in the same bucket, and it has to be + /// the bucket a collection treats conservatively. + #[test] + fn an_interface_says_whose_it_is_and_the_name_is_what_proves_it() { + let nic = identity("path-abc", "vm-1", 3); + let tap = tap_name(&nic); + let alias = interface_alias(&nic); + assert_eq!(alias, "dstack1:3:path-abc:vm-1"); + + let owner = owner_of(&tap, &alias).expect("its own record checks out"); + assert_eq!(owner.instance_id, "path-abc"); + assert_eq!(owner.vm_id, "vm-1"); + assert_eq!(owner.nic_index, 3); + + // A record naming some other interface proves nothing about this one. + // This is what makes the record unforgeable without making it + // authoritative: anything that can reach the socket can write a + // string, but only the true identity re-derives the name. + let forged = interface_alias(&identity("path-abc", "someone-elses-vm", 3)); + assert!(owner_of(&tap, &forged).is_none()); + assert!(owner_of(&tap, "").is_none()); + assert!(owner_of(&tap, "dstack1:3:path-abc").is_none()); + assert!(owner_of(&tap, &alias[..alias.len() - 2]).is_none()); + // A format this build does not know is not this format. + assert!(owner_of(&tap, &alias.replace("dstack1", "dstack2")).is_none()); + + // The two free-form fields are last and only the first of them has to + // be unambiguous, so a VM ID carrying the separator still reads back. + let odd = identity("path-abc", "vm:with:colons", 0); + assert_eq!( + owner_of(&tap_name(&odd), &interface_alias(&odd)).map(|owner| owner.vm_id), + Some("vm:with:colons".to_string()) + ); + // An instance ID carrying it is refused instead of mis-parsed. + assert!(validate_identity(&identity("path:abc", "vm-1", 0)).is_err()); + } + + /// An identity that cannot be recorded would produce an interface nothing + /// can attribute, which is the state this whole path exists to stop + /// creating. Refusing it is the only answer that keeps the invariant. + #[test] + fn an_identity_too_long_to_record_is_refused() { + let long = "v".repeat(128); + assert!(validate_identity(&identity("instance", &long, 0)).is_ok()); + let identity_too_long = identity(&"i".repeat(128), &long, 255); + assert!(interface_alias(&identity_too_long).len() > MAX_IFALIAS); + let error = validate_identity(&identity_too_long) + .unwrap_err() + .to_string(); + assert!(error.contains("too long to record"), "{error}"); + } + + /// The name space netd claims. A collection deletes what matches, so what + /// matches has to be exactly what netd can produce. + #[test] + fn the_managed_name_space_is_exactly_what_netd_produces() { + assert!(is_managed_name(&tap_name(&identity("instance", "vm", 0)))); + assert!(is_managed_name("dt0123456789ab")); + assert!(!is_managed_name("dt0123456789AB"), "digests are lower case"); + assert!(!is_managed_name("dt0123456789a"), "one short"); + assert!(!is_managed_name("dt0123456789abc"), "one long"); + assert!(!is_managed_name("dtzzzzzzzzzzzz")); + assert!(!is_managed_name("virbr0")); + assert!(!is_managed_name("eth0")); + // The whole space fits in IFNAMSIZ, or the kernel would refuse the + // names this reserves. + assert!(tap_name(&identity("instance", "vm", 255)).len() < 16); + } + + /// The command prints a table for a human and accepts no options to make it + /// print anything else, so this parses one. Narrowing to netd's own name + /// space is what makes that safe. + #[test] + fn the_binding_listing_reads_a_table_meant_for_a_person() { + let output = "\ + Port Dev Filter +--------------------------------- + dt1e053266e9f7 clean-traffic + dt28b105b3031a clean-traffic + vnet3 some-other-filter +"; + let names: HashSet = output + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|name| is_managed_name(name)) + .map(str::to_string) + .collect(); + assert_eq!(names.len(), 2); + assert!(names.contains("dt1e053266e9f7")); + // The header, the rule, and a binding that is not netd's all fall out. + assert!(!names.contains("Port")); + assert!(!names.contains("vnet3")); + } + + /// Everything else here reasons about strings. This puts the reasoning + /// next to the kernel: that an alias survives on a device netd actually + /// creates, that enumeration finds it, that the guards refuse what they + /// are meant to, and that removal leaves nothing. + /// + /// Refuses to run in the host's network namespace, so it cannot touch a + /// real node's interfaces even when it fails. Unsharing one from inside + /// the test is not enough: `/sys/class/net` keeps showing the old + /// namespace until sysfs is remounted, which is most of what `ip netns + /// exec` does. So it asks to be put in one: + /// + /// ```text + /// cargo test -p dstack-vmm --bins --no-run + /// sudo ip netns add dstack-netd-test + /// sudo ip netns exec dstack-netd-test \ + /// target/debug/deps/dstack_vmm- --ignored --test-threads=1 + /// sudo ip netns del dstack-netd-test + /// ``` + #[test] + #[ignore = "needs root and its own network namespace; see the doc comment"] + fn a_real_interface_carries_its_record_and_removal_leaves_nothing() { + assert!( + nix::unistd::Uid::effective().is_root(), + "this test needs root" + ); + let (mine, init) = ( + std::fs::read_link("/proc/self/ns/net").unwrap(), + std::fs::read_link("/proc/1/ns/net").unwrap(), + ); + assert_ne!( + mine, init, + "run this inside its own network namespace; it creates and deletes interfaces" + ); + // Nothing in this namespace to talk to, which is also the state of a + // node that does not filter: the listing has to work without libvirt. + let uri = "qemu:///nonexistent-for-this-test"; + + let nic = identity("test-instance", "vm-1", 2); + let tap = tap_name(&nic); + ip(&["tuntap", "add", "dev", &tap, "mode", "tap"]).unwrap(); + set_alias(&tap, &nic).unwrap(); + assert!(is_tuntap(&tap), "the kernel publishes tun_flags for a TAP"); + + let records = list_interfaces(uri, ""); + let record = records + .iter() + .find(|record| record.tap == tap) + .expect("an interface netd created is one netd can find"); + assert_eq!(record.instance_id.as_deref(), Some("test-instance")); + assert_eq!(record.vm_id.as_deref(), Some("vm-1")); + assert_eq!(record.nic_index, Some(2)); + assert_eq!(record.kind, "tap"); + // Narrowing by instance is what keeps one VMM's collection off + // another's interfaces. + assert_eq!(list_interfaces(uri, "test-instance").len(), 1); + assert!(list_interfaces(uri, "someone-else").is_empty()); + + // A device with one of netd's names that netd did not create. The name + // is 48 bits of digest, so this is not about collisions -- it is that + // `ip link delete` does not ask what it is deleting, and netd runs as + // root. + let impostor = tap_name(&identity("test-instance", "not-a-tap", 0)); + ip(&["link", "add", &impostor, "type", "dummy"]).unwrap(); + assert!(is_managed_name(&impostor)); + assert!( + !list_interfaces(uri, "") + .iter() + .any(|record| record.tap == impostor), + "a device netd did not create is not offered up for collection" + ); + let refused = remove_interface(uri, &impostor, BindingCleanup::Skip).unwrap_err(); + assert!(refused.to_string().contains("refusing to delete")); + + // An interface whose record does not re-derive its own name proves + // nothing, and lands in the same bucket as no record at all. + ip(&[ + "link", + "set", + "dev", + &tap, + "alias", + "dstack1:2:test-instance:some-other-vm", + ]) + .unwrap(); + let records = list_interfaces(uri, ""); + let record = records.iter().find(|record| record.tap == tap).unwrap(); + assert!(record.instance_id.is_none(), "a forged record is no record"); + + remove_interface(uri, &tap, BindingCleanup::Skip).unwrap(); + assert!(!Path::new("/sys/class/net").join(&tap).exists()); + assert!(!list_interfaces(uri, "") + .iter() + .any(|record| record.tap == tap)); + // Removing what is not there is not an error: a sweep derives names + // and most of them miss. + remove_interface(uri, &tap, BindingCleanup::Skip).unwrap(); + } + + /// Run with the same isolated-network-namespace setup as the test above. + #[test] + #[ignore = "needs root and its own network namespace"] + fn sweeps_without_libvirt_follow_the_nodes_filter_policy() { + assert!(nix::unistd::Uid::effective().is_root()); + assert_ne!( + std::fs::read_link("/proc/self/ns/net").unwrap(), + std::fs::read_link("/proc/1/ns/net").unwrap(), + "run this inside its own network namespace", + ); + let uri = "qemu:///nonexistent-for-this-test"; + for requires_binding in [false, true] { + let nic = identity("sweep-test", "vm-1", 0); + let tap = tap_name(&nic); + ip(&["tuntap", "add", "dev", &tap, "mode", "tap"]).unwrap(); + let mut config = NetdConfig { + libvirt_uri: uri.into(), + ..Default::default() + }; + config.network_filter = Some(NetworkFilterConfig { + mode: if requires_binding { + crate::config::NetworkFilterMode::Libvirt + } else { + crate::config::NetworkFilterMode::None + }, + ..Default::default() + }); + let sweep = || { + sweep_vm_interfaces( + &config.libvirt_uri, + &nic.instance_id, + &nic.vm_id, + config.filter_policy().requires_binding(), + ) + }; + let result = sweep(); + // Even a binding failure must not leave the TAP on the bridge. + assert!(!Path::new("/sys/class/net").join(&tap).exists()); + if requires_binding { + assert!(result.is_err()); + // A retry cannot claim success just because the TAP is gone: + // its binding may still exist in the unreachable libvirt. + assert!(sweep().is_err()); + } else { + assert_eq!(result.unwrap(), 1); + assert_eq!(sweep().unwrap(), 0); + } + } + } + + /// Run with the same isolated-network-namespace setup as the tests above. + /// A manual removal reports success only once nothing at the name is left + /// that the node's policy needs confirmed. + #[test] + #[ignore = "needs root and its own network namespace"] + fn a_manual_removal_does_not_claim_a_binding_it_could_not_confirm() { + assert!(nix::unistd::Uid::effective().is_root()); + assert_ne!( + std::fs::read_link("/proc/self/ns/net").unwrap(), + std::fs::read_link("/proc/1/ns/net").unwrap(), + "run this inside its own network namespace", + ); + let uri = "qemu:///nonexistent-for-this-test"; + let tap = tap_name(&identity("manual-test", "vm-1", 0)); + + // Nothing but a possible binding at the name, and no libvirt to ask. + let error = remove_interface_by_name(uri, &tap, false).unwrap_err(); + assert!(format!("{error:#}").contains("cannot confirm"), "{error:#}"); + + for requires_binding in [false, true] { + ip(&["tuntap", "add", "dev", &tap, "mode", "tap"]).unwrap(); + let result = remove_interface_by_name(uri, &tap, requires_binding); + assert!( + !Path::new("/sys/class/net").join(&tap).exists(), + "the interface goes whatever happened to its binding" + ); + assert_eq!(result.is_err(), requires_binding, "{result:?}"); + } + + let impostor = tap_name(&identity("manual-test", "not-a-tap", 0)); + ip(&["link", "add", &impostor, "type", "dummy"]).unwrap(); + let refused = remove_interface_by_name(uri, &impostor, false).unwrap_err(); + assert!(refused.to_string().contains("refusing to delete")); + ip(&["link", "delete", "dev", &impostor]).unwrap(); + assert!(remove_interface_by_name(uri, "eth0", false).is_err()); } /// The VMM tells "netd is not running" apart from "netd refused". Both diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 41674726c..70d399729 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -882,6 +882,11 @@ def create_app_compose(self, args) -> None: "no_instance_id": args.no_instance_id, "secure_time": args.secure_time, } + # Only emit the field when it is turned off, so a default compose file + # keeps the exact bytes -- and therefore the compose hash -- it had + # before this option existed. + if args.no_storage_discard: + app_compose["storage_discard"] = False if args.key_provider: app_compose["key_provider"] = args.key_provider if args.prelaunch_script: @@ -954,6 +959,8 @@ def create_vm(self, args) -> None: "stopped": args.stopped, "no_tee": args.no_tee, } + if args.disk_prealloc: + params["disk_prealloc"] = args.disk_prealloc if args.simulated_tee: params["simulated_tee"] = args.simulated_tee if args.swap is not None: @@ -1373,7 +1380,9 @@ def show_info(self, vm_id: str, json_output: bool = False) -> None: print(f"Image Version: {info.get('image_version', '-')}") print(f"vCPU: {config.get('vcpu', '-')}") print(f"Memory: {config.get('memory', '-')}MB") - print(f"Disk: {config.get('disk_size', '-')}GB") + prealloc = config.get("disk_prealloc") or "off" + disk_note = "" if prealloc == "off" else f" (prealloc: {prealloc})" + print(f"Disk: {config.get('disk_size', '-')}GB{disk_note}") print(f"GPUs: {self._format_gpu_info(config.get('gpus'))}") print(f"Boot Progress: {info.get('boot_progress', '-')}") if info.get("boot_error"): @@ -1909,6 +1918,15 @@ def _patched_format_help(): compose_parser.add_argument( "--secure-time", action="store_true", help="Enable secure time" ) + compose_parser.add_argument( + "--no-storage-discard", + action="store_true", + help=( + "Keep deleted data-disk blocks instead of returning them to the " + "host. Required by 'deploy --disk-prealloc falloc|full', and it " + "also stops the host from seeing filesystem allocation patterns" + ), + ) compose_parser.add_argument( "--swap", type=parse_memory_size, @@ -1979,6 +1997,19 @@ def _patched_format_help(): deploy_parser.add_argument( "--hugepages", action="store_true", help="Enable hugepages for the VM" ) + deploy_parser.add_argument( + "--disk-prealloc", + choices=["off", "metadata", "falloc", "full"], + default=None, + help=( + "Preallocate the data disk on the host: metadata (qcow2 metadata " + "only), falloc (reserve space, fast) or full (reserve and zero). " + "falloc and full reserve host blocks and require " + "storage_discard=false in the compose file (see " + "'compose --no-storage-discard'). Defaults to the VMM's " + "cvm.disk_prealloc setting" + ), + ) deploy_parser.add_argument("--kms-url", action="append", type=str, help="KMS URL") deploy_parser.add_argument( "--kms-encrypt-url", diff --git a/dstack/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts index 4334d8844..8325a6210 100644 --- a/dstack/vmm/ui/src/components/CreateVmDialog.ts +++ b/dstack/vmm/ui/src/components/CreateVmDialog.ts @@ -61,6 +61,13 @@ const CreateVmDialogComponent = { return `queues: auto${cap}`; }, }, + computed: { + // Only falloc and full reserve host blocks; metadata does not, so it does + // not conflict with discard and the VMM accepts it either way. + reservesDiskBlocks(): boolean { + return ['falloc', 'full'].includes((this as any).form.disk_prealloc); + }, + }, emits: ['close', 'submit', 'load-compose'], template: /* html */ `
@@ -116,6 +123,20 @@ const CreateVmDialogComponent = {
+
+ + + Turn Storage Discard off, or this deployment is rejected: the guest would return the reserved blocks to the host on its first trim. +
+
+
Total Instances: {{ totalVMs }} diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index 068cece16..1cf52e0c0 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -46,7 +46,11 @@ max_allocable_memory_in_mb = 100_000 # MB qmp_socket = false # The user to run the VM as. If empty, the VM will be run as the current user. # Unique namespace when multiple dstack-vmm instances share one host. When -# empty, a stable value is derived from run_path. +# empty, a stable value is derived from run_path. It is what netd records on +# every host interface this VMM asks for, and the name space those interfaces' +# names are derived in. Two VMMs on one host must not share one: each would +# build interfaces at names the other can also produce. Left empty it is +# derived from run_path, which cannot collide. May not contain ":". instance_id = "" # Network choices that deployment RPC callers may make. Macvtap is excluded # by default because libvirt nwfilter applies only to bridge interfaces. @@ -101,6 +105,31 @@ tdx_attestation_variant = "auto" host_share_mode = "9p" +# Default preallocation for a new CVM data disk. This is a host storage +# policy -- the guest cannot see it and it does not affect measurements -- and +# a deploy request may pick another mode per VM. +# "off" : allocate nothing; the image grows as the guest writes, so the +# host is oversubscribed and a guest write can hit ENOSPC. +# "metadata" : allocate qcow2 metadata only; no space is reserved for data. +# "falloc" : reserve the whole disk with fallocate(2). Fast, and it turns a +# full host into a VM start failure instead of a guest ENOSPC. +# "full" : reserve and zero the whole disk. Creation writes the full disk +# size, so a large disk takes minutes. +# "falloc" and "full" reserve host blocks, so they require +# `storage_discard: false` in the app's app-compose.json -- a guest that may +# discard hands the reserved blocks straight back -- and deployments that ask +# for both are rejected. Note what that costs the app: editing app-compose +# changes its hash and therefore its app id, so those two modes are not free +# of the identity for an app that ships with discard on. "metadata" reserves +# nothing and needs no compose change. +# Preallocation on top of a base image needs qcow2 subcluster allocation, so +# the VMM creates such disks with extended_l2=on (QEMU 5.2+). +# This value applies to disks created from here on, and a VM's mode is fixed +# at deployment. An existing data disk is never converted -- editing the VM's +# vm-manifest.json only changes what a later resize does -- so the way to +# preallocate an existing VM is to redeploy it. +disk_prealloc = "off" + # QGS (Quote Generation Service) vsock port for kernel-level TSM support. # When set, QEMU will pass this port to tdx-guest for configfs-tsm quote generation. # The guest kernel will use this vsock port to communicate with the host QGS. @@ -250,8 +279,9 @@ include = [] # Allow attach all GPUs allow_attach_all = true # Issue a Secondary Bus Reset on each GPU's dedicated upstream PCIe bridge -# immediately before QEMU attaches the device. This clears stale GPU FSP/SPDM -# state left by a VM that terminated during GPU firmware initialization. +# immediately before QEMU attaches the device, using a VFIO PCI hot reset so +# that no root privileges are needed. This clears stale GPU FSP/SPDM state +# left by a VM that terminated during GPU firmware initialization. sanitize_on_attach = true # Maximum time for all reset GPUs to become continuously VFIO-ready. sbr_timeout_ms = 10000 diff --git a/os/common/rootfs/dstack-prepare.sh b/os/common/rootfs/dstack-prepare.sh index d6527d229..40440d3c7 100755 --- a/os/common/rootfs/dstack-prepare.sh +++ b/os/common/rootfs/dstack-prepare.sh @@ -336,6 +336,10 @@ docker-compose|nerdctl-compose) if [[ ! -f docker-compose.yaml ]]; then jq -r '.docker_compose_file' app-compose.json >docker-compose.yaml fi + # Failures are tolerated on purpose: this is a cleanup pass, and the + # command refuses to act on a compose file whose service list it cannot + # enumerate (`include:`) rather than mistake a live service for an orphan. + # The error still reaches the journal. dstack-util remove-orphans --no-dockerd -f docker-compose.yaml || true ;; esac diff --git a/os/common/rootfs/tdx-guest-tune.sh b/os/common/rootfs/tdx-guest-tune.sh new file mode 100755 index 000000000..1ade51b7b --- /dev/null +++ b/os/common/rootfs/tdx-guest-tune.sh @@ -0,0 +1,157 @@ +#!/bin/sh +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# Runtime scaling policy for many-vCPU TDX guests (dstack kernel with +# tdx_wake_q_batch, tdx_pv_single_ipi and CONFIG_HALTPOLL_CPUIDLE=m). +# +# Experimental helper that may be removed from future images; the stable +# interface is the kernel switches it sets. See docs/tdx-guest-scaling.md. +# +# Usage: /usr/lib/dstack/tdx-guest-tune.sh [throughput|default|off|status] [options] +# throughput guest halt polling + wake-queue IPI batching (highest throughput, +# extra CPU while idle-polling; for vCPUs that are not overcommitted) +# default kernel defaults: no guest polling, wake-queue batching and PV +# single-target IPIs on ("nopoll" is accepted as an alias) +# off all switches off and polling module unloaded (pre-patch behavior) +# status print the current state +# Options: +# --poll-us N guest halt-poll window in microseconds (default: by vCPU count) +# --pv-single Y|N override tdx_pv_single_ipi for the selected profile +# --force apply even if this is not a TDX guest +# Environment: TDX_TUNE_PROFILE, TDX_TUNE_POLL_US, TDX_TUNE_PV_SINGLE. +set -eu + +KP=/sys/module/kernel/parameters +HP=/sys/module/haltpoll/parameters +IDLE=/sys/devices/system/cpu/cpuidle + +profile=${TDX_TUNE_PROFILE:-throughput} +[ "$profile" = nopoll ] && profile=default +poll_us=${TDX_TUNE_POLL_US:-} +pv_single=${TDX_TUNE_PV_SINGLE:-} +force=0 + +log() { echo "tdx-guest-tune: $*"; } +die() { echo "tdx-guest-tune: $*" >&2; exit 1; } + +while [ $# -gt 0 ]; do + case $1 in + throughput|default|off|status) profile=$1 ;; + nopoll) profile=default ;; + --poll-us) [ $# -ge 2 ] || die "--poll-us needs a value"; poll_us=$2; shift ;; + --pv-single) [ $# -ge 2 ] || die "--pv-single needs Y or N"; pv_single=$2; shift ;; + --force) force=1 ;; + -h|--help) sed -n '12,23p' "$0"; exit 0 ;; + *) die "unknown argument: $1" ;; + esac + shift +done + +readf() { cat "$1" 2>/dev/null || echo "-"; } + +status() { + driver=$(readf "$IDLE/current_driver") + governor=$(readf "$IDLE/current_governor") + loaded=no + grep -q '^cpuidle_haltpoll ' /proc/modules 2>/dev/null && loaded=yes + log "vcpus=$(nproc) tdx_guest=$(is_tdx && echo yes || echo no) idle_driver=$driver governor=$governor haltpoll_module=$loaded" + log "guest_halt_poll_ns=$(readf "$HP/guest_halt_poll_ns") allow_shrink=$(readf "$HP/guest_halt_poll_allow_shrink") grow_start=$(readf "$HP/guest_halt_poll_grow_start")" + log "tdx_wake_q_batch=$(readf "$KP/tdx_wake_q_batch") tdx_pv_single_ipi=$(readf "$KP/tdx_pv_single_ipi")" +} + +is_tdx() { + [ -e /dev/tdx_guest ] || grep -qw tdx_guest /proc/cpuinfo 2>/dev/null +} + +write() { # write FILE VALUE + [ -w "$1" ] || die "missing or read-only: $1 (kernel without the dstack TDX knobs?)" + printf '%s\n' "$2" > "$1" +} + +# Window derived from measurements: 32 vCPUs -> 200 us, 248 vCPUs -> 4 ms. +# Both are ~20-25x the TD exit round trip seen under that vCPU concurrency +# (about 9 us and 170 us); intermediate steps are interpolated, not measured. +default_poll_us() { + n=$(nproc) + if [ "$n" -le 32 ]; then echo 200 + elif [ "$n" -le 64 ]; then echo 500 + elif [ "$n" -le 128 ]; then echo 1000 + else echo 4000 + fi +} + +# Enable guest halt polling with a fixed, non-shrinking window. +enable_polling() { + ns=$(( $1 * 1000 )) + # Clamp any previously grown per-CPU limit: allow shrinking to zero on the + # next long idle, then set the new window before disabling shrink again. + write "$HP/guest_halt_poll_allow_shrink" Y + write "$HP/guest_halt_poll_shrink" 0 + write "$HP/guest_halt_poll_ns" 1000 + sleep 1 + write "$HP/guest_halt_poll_shrink" 2 + write "$HP/guest_halt_poll_grow_start" "$ns" + write "$HP/guest_halt_poll_ns" "$ns" + write "$HP/guest_halt_poll_allow_shrink" N + if [ "$(readf "$IDLE/current_driver")" != haltpoll ]; then + modprobe cpuidle_haltpoll force=1 || + die "cannot load cpuidle_haltpoll (image without CONFIG_HALTPOLL_CPUIDLE=m?)" + fi +} + +# Restore the haltpoll governor parameters to their kernel defaults. +reset_polling() { + write "$HP/guest_halt_poll_allow_shrink" Y + write "$HP/guest_halt_poll_grow_start" 50000 + write "$HP/guest_halt_poll_ns" 200000 +} + +disable_polling() { + if grep -q '^cpuidle_haltpoll ' /proc/modules 2>/dev/null; then + rmmod cpuidle_haltpoll + elif [ "$(readf "$IDLE/current_driver")" = haltpoll ]; then + log "warning: haltpoll is built in or bound at boot; only shortening its window" + write "$HP/guest_halt_poll_allow_shrink" Y + write "$HP/guest_halt_poll_ns" 50000 + fi +} + +case $profile in + status) status; exit 0 ;; + throughput|default|off) ;; + *) die "unknown profile: $profile" ;; +esac + +[ "$(id -u)" -eq 0 ] || die "must run as root" +if ! is_tdx && [ "$force" -ne 1 ]; then + die "not a TDX guest; use --force to apply anyway" +fi +case ${pv_single:-} in ""|Y|N) ;; *) die "--pv-single must be Y or N" ;; esac +case ${poll_us:-} in ""|*[!0-9]*) [ -z "${poll_us:-}" ] || die "--poll-us must be an integer" ;; esac + +case $profile in + throughput) + [ -n "${poll_us:-}" ] || poll_us=$(default_poll_us) + enable_polling "$poll_us" + write "$KP/tdx_wake_q_batch" Y + # With polling and batching, PV single-target IPIs measured -5%..+5%. + write "$KP/tdx_pv_single_ipi" "${pv_single:-N}" + ;; + default) + disable_polling + reset_polling + write "$KP/tdx_wake_q_batch" Y + # Without polling most wakeups need an IPI; PV single IPIs measured +10%..+32%. + write "$KP/tdx_pv_single_ipi" "${pv_single:-Y}" + ;; + off) + disable_polling + reset_polling + write "$KP/tdx_wake_q_batch" N + write "$KP/tdx_pv_single_ipi" "${pv_single:-N}" + ;; +esac +log "applied profile: $profile" +status diff --git a/os/image/README.md b/os/image/README.md index b93a39883..d1f4e1256 100644 --- a/os/image/README.md +++ b/os/image/README.md @@ -47,6 +47,13 @@ the helper lives beside the common assembler. OCI registry. It is likewise independent of the backend that produced the image. +## Kernel build tree + +The optional `artifacts.kernel_devel` manifest entry is published as +`--kernel-devel.tar.gz` beside the image archives. It is not in +`sha256sum.txt`, so it does not affect `os_image_hash`. `kernel-builder/` +builds the matching module-builder container image. + ## Kernel setup-header normalization `assemble.sh` runs `normalize-kernel-header.py` over `bzImage` before it @@ -64,36 +71,61 @@ host's QEMU version — and the host is the one that declares that version. The fix has two halves that must stay in sync: -- this script zeroes those fields in the kernel we ship; -- `0007-OvmfPkg-QemuKernelLoaderFsDxe-normalize-setup-header.patch` zeroes them - again in OVMF, before the kernel blob is measured and loaded. +- this script fills in those fields in the kernel we ship, with the values + QEMU <= 10.1 would write; +- `0007-OvmfPkg-QemuKernelLoaderFsDxe-normalize-setup-header.patch` writes the + same values in OVMF, before the kernel blob is measured and loaded. The result is that RTMR[1] is the plain Authenticode hash of `bzImage` as listed in `sha256sum.txt`, on every QEMU version and at every guest memory size. +Normalizing to a header QEMU once wrote, rather than to zeros, is what keeps an +earlier release able to verify these images: `dstack-mr` already computes that +header (`patch_kernel`) for an image that does not declare the flag, so a 0.5.x +KMS can still onboard a 0.6.0 root with image verification on. That +recomputation takes the guest RAM size as an input, so it reproduces the +normalized bytes only for guests of exactly 2 GiB or at least 2816 MiB. Below +2 GiB, and between 2 GiB and 2816 MiB, QEMU placed the initrd somewhere else, +so such a verifier computes a different RTMR[1]; the no-image-download path +refuses that same range for the same reason. The shipped initrd's size is an +input too, since it decides `ramdisk_image`: QEMU packs the initrd against a +ceiling, which is 4G for a kernel that sets `XLF_CAN_BE_LOADED_ABOVE_4G` +(bit 1 of `xloadflags` — bit 6, `0x40`, is `XLF_5LEVEL_ENABLED`, a different +flag) and `0x37ffffff` for one that does not, then clamped to the below-4G +window QEMU leaves free of ACPI tables. + +The kernel must be loaded high, which every kernel this build ships is. QEMU +puts a low-loaded kernel's command line at `0x9a000 - cmdline_size`, a value +that depends on the command line the host passes, so there is nothing to +normalize to: the script refuses such a kernel and OVMF leaves its header as +served. + `assemble.sh` records this in `metadata.json` as `"kernel_header_normalized": -true`, and re-runs the script with `--check` first so the build fails rather -than shipping a kernel that disagrees with what the image declares. Images -without the field are the ones built before this existed; `dstack-mr` measures -those the old way, against QEMU's rewritten header. - -The field set comes from the boot protocol, not from QEMU's behavior: every -field `Documentation/arch/x86/boot.rst` types as `write` is one the boot loader -fills in and the kernel supplies no value for. Fields typed `modify` carry real -kernel-supplied values — `code32_start` is the protected-mode entry point — and -are deliberately left alone. - -In practice this rewrites **two bytes**: `heap_end_ptr` (0x224) is the only -`write` field a built kernel leaves non-zero. That is safe for every boot path: -the boot protocol types it `write (obligatory)`, `init_heap()` reads it only -when the boot loader has set `CAN_USE_HEAP` (which the kernel builds clear and -this script also clears), and on the EFI-stub path the real-mode setup code -never runs at all. The PE headers sit at 0x40..0x170, so no setup-header field -overlaps them and the EFI entry point is untouched. +true`. What makes that declaration true is the OVMF half, which the same build +applies -- `ovmf-build.sh` and the bitbake recipe both fail if the patch does +not apply, so an image cannot ship the flag with firmware that ignores it. +Images without the field are the ones built before this existed; `dstack-mr` +measures those the old way, against QEMU's rewritten header. + +What gets written is one fixed header, not a reimplementation of QEMU's loader: +every field written is one QEMU fills in as boot loader, and the script refuses +any kernel outside the shape it reproduces. All but one of those fields are +typed `write` in `Documentation/arch/x86/boot.rst`, meaning the boot loader +supplies them and the kernel has no value there. The exception is `loadflags`, +typed `modify (obligatory)`, of which only the `CAN_USE_HEAP` bit is set — a +bit the protocol assigns to the boot loader. Fields that carry real +kernel-supplied values, such as `code32_start`, the protected-mode entry point, +are deliberately left alone, and so are the `write` fields QEMU never touches. + +That is safe for every boot path: these are the values QEMU itself wrote for +every release before 10.2, and on the EFI-stub path the real-mode setup code +never runs at all -- the stub takes the initrd through LoadFile2 and the +command line through its load options. The PE headers sit at 0x40..0x170, so no +setup-header field overlaps them and the EFI entry point is untouched. To check an image without modifying it: ```bash -./normalize-kernel-header.py --check /path/to/bzImage +./normalize-kernel-header.py --check /path/to/bzImage /path/to/initramfs.cpio.gz ``` diff --git a/os/image/assemble.sh b/os/image/assemble.sh index b27c5a880..aa7e11b34 100755 --- a/os/image/assemble.sh +++ b/os/image/assemble.sh @@ -109,6 +109,7 @@ exact_keys( data["artifacts"], "artifacts", ("initramfs", "kernel", "firmware", "rootfs_verity", "firmware_sev", "uki"), + ("kernel_devel",), ) if "backend_metadata" in data and not isinstance(data["backend_metadata"], dict): raise SystemExit("manifest field must be an object: backend_metadata") @@ -179,6 +180,7 @@ values = [ artifact("rootfs_verity"), artifact("firmware_sev", optional=True), artifact("uki", optional=True), + artifact("kernel_devel", optional=True), ] for value in values: if not isinstance(value, str): @@ -187,7 +189,7 @@ for value in values: PYMANIFEST ) -if [ "${#MANIFEST_VALUES[@]}" -ne 15 ]; then +if [ "${#MANIFEST_VALUES[@]}" -ne 16 ]; then echo "Error: failed to read artifact manifest: $MANIFEST" >&2 exit 1 fi @@ -211,6 +213,7 @@ OVMF_FIRMWARE=${MANIFEST_VALUES[11]} ROOTFS_IMAGE=${MANIFEST_VALUES[12]} OVMF_SEV_FIRMWARE=${MANIFEST_VALUES[13]} UKI_IMAGE=${MANIFEST_VALUES[14]} +KERNEL_DEVEL_ARCHIVE=${MANIFEST_VALUES[15]} SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) REPO_ROOT=$(realpath "$SCRIPT_DIR/../..") @@ -230,6 +233,7 @@ case "$OUTPUT_DIR" in esac IMAGE_TAR=${IMAGE_TAR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}.tar.gz"} IMAGE_TAR_UKI=${IMAGE_TAR_UKI:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}-uki.tar.gz"} +KERNEL_DEVEL_TAR=${KERNEL_DEVEL_TAR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}-kernel-devel.tar.gz"} TAR_DIR_NAME=$(basename "$OUTPUT_DIR") echo "Assembling ${DIST_NAME} ${DSTACK_VERSION} from ${BACKEND} artifacts (${FLAVOR})" @@ -416,7 +420,8 @@ verbose cp "$KERNEL_IMAGE" "${OUTPUT_DIR}/bzImage" # result into RTMR[1]. QEMU >= 10.2 stopped doing that for confidential guests, # so leaving the header as built would make the same image measure differently # per QEMU version. Normalizing here, and again in OVMF before it measures, -# makes RTMR[1] the plain Authenticode hash of this file. Runs before +# makes RTMR[1] the plain Authenticode hash of this file. The initrd size is an +# input, so this runs once both files are in place, and before # tdx-measurement-cbor and sha256sum.txt below, so both cover the normalized # kernel. The matching half is in OVMF: metadata.json declares # kernel_header_normalized below, and what makes that declaration true is @@ -424,7 +429,7 @@ verbose cp "$KERNEL_IMAGE" "${OUTPUT_DIR}/bzImage" # same build applies -- ovmf-build.sh and the bitbake recipe both fail if it # does not apply. See os/image/README.md. verbose "$(dirname "${BASH_SOURCE[0]}")/normalize-kernel-header.py" \ - "${OUTPUT_DIR}/bzImage" + "${OUTPUT_DIR}/bzImage" "${OUTPUT_DIR}/initramfs.cpio.gz" verbose cp "$OVMF_FIRMWARE" "${OUTPUT_DIR}/ovmf.fd" # AMD SEV firmware (additive). Shipped alongside the TDX firmware so a SEV-SNP @@ -664,3 +669,9 @@ if [ "$DSTACK_TAR_RELEASE" = "1" ]; then echo fi fi + +# Publish the optional developer artifact beside the images, without adding it +# to sha256sum.txt or os_image_hash. See docs/building-guest-os.md. +if [ -n "$KERNEL_DEVEL_ARCHIVE" ]; then + cp "$KERNEL_DEVEL_ARCHIVE" "$KERNEL_DEVEL_TAR" +fi diff --git a/os/image/kernel-builder/Dockerfile b/os/image/kernel-builder/Dockerfile new file mode 100644 index 000000000..31b2d7f23 --- /dev/null +++ b/os/image/kernel-builder/Dockerfile @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 +# +# Toolchain image for building out-of-tree modules against one dstack guest OS +# kernel: its exported build tree plus a compiler from the same pinned Debian +# snapshot the kernel was built with. Built by build.sh. + +ARG BASE_IMAGE=debian:trixie-slim +FROM ${BASE_IMAGE} + +ARG DEBIAN_SNAPSHOT +ARG KERNEL_RELEASE +ARG IMAGE_VERSION=unknown +ARG GIT_REVISION=unknown + +# Pin apt to the snapshot; trixie also ships deb822 sources that must go. +RUN set -eux; \ + suite=$(. /etc/os-release && echo "${VERSION_CODENAME}"); \ + rm -f /etc/apt/sources.list /etc/apt/sources.list.d/*; \ + echo "deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT:?} ${suite} main" \ + > /etc/apt/sources.list; \ + apt-get update; \ + apt-get install --yes --no-install-recommends \ + bc bison build-essential flex kmod libelf-dev libssl-dev; \ + rm -rf /var/lib/apt/lists/* + +ENV KERNEL_RELEASE=${KERNEL_RELEASE} +ENV KDIR=/usr/src/linux-headers-${KERNEL_RELEASE} + +ADD kernel-devel.tar.gz /tmp/ +RUN set -eux; \ + mv /tmp/*-kernel-devel/linux-headers-* /usr/src/; \ + rm -rf /tmp/*-kernel-devel; \ + test "$(cat "$KDIR/include/config/kernel.release")" = "${KERNEL_RELEASE:?}"; \ + mkdir -p "/lib/modules/${KERNEL_RELEASE}"; \ + ln -s "$KDIR" "/lib/modules/${KERNEL_RELEASE}/build" + +LABEL org.opencontainers.image.title="dstack kernel module builder" \ + org.opencontainers.image.source="https://github.com/Dstack-TEE/dstack" \ + org.opencontainers.image.revision="${GIT_REVISION}" \ + org.opencontainers.image.version="${IMAGE_VERSION}" \ + network.phala.dstack.kernel-release="${KERNEL_RELEASE}" + +WORKDIR /work diff --git a/os/image/kernel-builder/build.sh b/os/image/kernel-builder/build.sh new file mode 100755 index 000000000..b179f976d --- /dev/null +++ b/os/image/kernel-builder/build.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 +# Usage: build.sh KERNEL_DEVEL_ARCHIVE IMAGE_REF +set -euo pipefail + +archive=$(realpath "${1:?kernel-devel archive required}") +image=${2:?image ref required} +SELF=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../../mkosi/versions.env +source "$SELF/../../mkosi/versions.env" + +# Keep the build context to the one archive. +context=$(mktemp -d) +trap 'rm -rf "$context"' EXIT +cp "$archive" "$context/kernel-devel.tar.gz" + +${DOCKER:-docker} build \ + --file "$SELF/Dockerfile" \ + --build-arg "BASE_IMAGE=debian:$DEBIAN_RELEASE-slim" \ + --build-arg "DEBIAN_SNAPSHOT=$DEBIAN_SNAPSHOT" \ + --build-arg "KERNEL_RELEASE=$KERNEL_VERSION-dstack" \ + --build-arg "IMAGE_VERSION=${IMAGE_VERSION:-$DSTACK_VERSION}" \ + --build-arg "GIT_REVISION=${GIT_REVISION:-unknown}" \ + --tag "$image" \ + "$context" diff --git a/os/image/normalize-kernel-header.py b/os/image/normalize-kernel-header.py index d628e5f4f..01509fe93 100755 --- a/os/image/normalize-kernel-header.py +++ b/os/image/normalize-kernel-header.py @@ -9,53 +9,85 @@ whether QEMU rewrote the header -- which changed in QEMU 10.2 (commit a7542a38f399, "x86/loader: Don't update kernel header for CoCo VMs"). -Zeroing those fields in the shipped kernel, and having OVMF zero them again +Writing those fields in the shipped kernel, and having OVMF write them again before measuring, makes RTMR[1] the plain Authenticode hash of the file we ship, on every QEMU version. -The field set comes from the boot protocol, not from QEMU: every field -`Documentation/arch/x86/boot.rst` types as `write` is one the boot loader -fills in and the kernel supplies no value for. Fields typed `modify` carry -real kernel-supplied values (`code32_start` is the protected-mode entry point) -and are left alone. - -In a freshly built kernel every one of these fields is already zero except -`heap_end_ptr`, so this normally rewrites exactly two bytes. +The values are the ones QEMU <= 10.1 wrote for a kernel loaded high in a guest +with 2 GiB or more of RAM below 4G. Zeros would serve the measurement goal +equally well; these particular values are chosen because `dstack-mr` already +computes them (`patch_kernel` in dstack/dstack-mr/src/kernel.rs) for an image +that does not declare `kernel_header_normalized`, so a verifier or KMS from an +earlier dstack release can still measure these images. + +This writes one fixed header. It is not a reimplementation of QEMU's loader: +the branches below reproduce the header QEMU in fact wrote for the kernels +dstack ships, and a kernel outside that shape is refused rather than guessed +at. + +Every field written is one QEMU fills in as boot loader. All but one are typed +`write` in `Documentation/arch/x86/boot.rst`: the boot loader supplies them and +the kernel has no value there. The exception is `loadflags`, typed +`modify (obligatory)`, of which only `CAN_USE_HEAP` is set -- a bit the +protocol assigns to the boot loader. Fields carrying real kernel-supplied +values, such as `code32_start`, are left alone, and so are the `write` fields +QEMU never touches: a built kernel ships them zero, so writing them would only +add a way to disagree. + +The initrd size decides `ramdisk_image`, so the initrd the image ships is an +input. """ import argparse import sys -# Offset, size, name -- every boot.rst field typed `write`. -WRITE_FIELDS = [ - (0x210, 1, "type_of_loader"), - (0x218, 4, "ramdisk_image"), - (0x21C, 4, "ramdisk_size"), - (0x224, 2, "heap_end_ptr"), - (0x226, 1, "ext_loader_ver"), - (0x227, 1, "ext_loader_type"), - (0x228, 4, "cmd_line_ptr"), - (0x23C, 4, "hardware_subarch"), - (0x240, 8, "hardware_subarch_data"), - (0x250, 8, "setup_data"), -] +# Offsets of the boot.rst `write` fields QEMU fills in. +TYPE_OF_LOADER_OFFSET = 0x210 +RAMDISK_IMAGE_OFFSET = 0x218 +RAMDISK_SIZE_OFFSET = 0x21C +HEAP_END_PTR_OFFSET = 0x224 +CMD_LINE_PTR_OFFSET = 0x228 +INITRD_ADDR_MAX_OFFSET = 0x22C +XLOADFLAGS_OFFSET = 0x236 LOADFLAGS_OFFSET = 0x211 +LOADED_HIGH = 0x01 CAN_USE_HEAP = 0x80 +# xloadflags bit 1. Bit 6, 0x40, is XLF_5LEVEL_ENABLED, not this flag. +XLF_CAN_BE_LOADED_ABOVE_4G = 0x02 + +# What QEMU writes: "Qemu" version 0, the real-mode block and command line it +# loads a kernel loaded high at, the 0x200-byte gap it leaves below the command +# line for the setup heap, the below-4G window it reserves for ACPI tables with +# RAM split at 2 GiB, the initrd ceiling a kernel that declares none gets, and +# the alignment it rounds the initrd address down to. +TYPE_OF_LOADER_QEMU = 0xB0 +REAL_ADDR = 0x10000 +CMDLINE_ADDR = 0x20000 +SETUP_HEAP_GAP = 0x200 +LOW_MEMORY_SPLIT = 0x80000000 +ACPI_DATA_SIZE = 0x28000 +INITRD_ADDR_MAX_DEFAULT = 0x37FFFFFF +INITRD_ALIGNMENT = 0x1000 HEADER_MAGIC_OFFSET = 0x202 HEADER_MAGIC = b"HdrS" VERSION_OFFSET = 0x206 -# One past the last field this touches. Anything shorter cannot carry a setup -# header, and slicing past the end would silently grow the image instead of -# failing. The OVMF side applies the same bound. +# Minimum length for an image to be treated as carrying a setup header. The +# fields below all sit under 0x238; 0x258 is the end of the protocol 2.09 +# header, kept as the bound so a truncated image is rejected outright instead +# of being sliced past its end, which would silently grow it. The OVMF side +# applies the same bound. HEADER_END_OFFSET = 0x258 -# `setup_data` (0x250) requires 2.09+; every field above exists by then. -MIN_PROTOCOL = 0x0209 +# `cmd_line_ptr` moved into the setup header in 2.02, `initrd_addr_max` +# arrived in 2.03 and `xloadflags` in 2.12. +MIN_PROTOCOL = 0x0202 +IAM_PROTOCOL = 0x0203 +XLF_PROTOCOL = 0x020C -def normalize(image: bytearray) -> list: - """Zero the boot-loader-written fields. Returns the fields it changed.""" +def target_fields(image: bytes, initrd_size: int) -> list: + """Return the (offset, bytes, name) QEMU writes for this kernel.""" if len(image) < HEADER_END_OFFSET: raise ValueError( f"image is {len(image)} bytes, shorter than the " @@ -71,25 +103,79 @@ def normalize(image: bytearray) -> list: "script normalizes is not guaranteed" ) - changed = [] - for offset, size, name in WRITE_FIELDS: - old = bytes(image[offset : offset + size]) - if old != bytes(size): - changed.append((name, offset, old.hex(), "0" * (size * 2))) - image[offset : offset + size] = bytes(size) - - loadflags = image[LOADFLAGS_OFFSET] - if loadflags & CAN_USE_HEAP: - changed.append( - ( - "loadflags", - LOADFLAGS_OFFSET, - f"{loadflags:02x}", - f"{loadflags & ~CAN_USE_HEAP:02x}", - ) + if not image[LOADFLAGS_OFFSET] & LOADED_HIGH: + # For a kernel loaded low, QEMU's command line sits at + # 0x9a000 - cmdline_size, which depends on the command line the host + # passes: there is no single value to normalize to. Every kernel this + # build ships is loaded high, so refuse rather than write a value QEMU + # never writes. The OVMF side leaves such a header as served. + raise ValueError( + "the kernel is not loaded high; its cmd_line_ptr depends on the " + "command line and cannot be normalized" + ) + + fields = [ + (TYPE_OF_LOADER_OFFSET, bytes([TYPE_OF_LOADER_QEMU]), "type_of_loader"), + ( + LOADFLAGS_OFFSET, + bytes([image[LOADFLAGS_OFFSET] | CAN_USE_HEAP]), + "loadflags", + ), + # heap_end_ptr is two bytes, but the value never exceeds 16 bits and + # the two bytes above it (ext_loader_ver, ext_loader_type) are zero in + # a built kernel and untouched by QEMU, so writing the word + # zero-extended to 32 bits is what OVMF and dstack-mr also write. + ( + HEAP_END_PTR_OFFSET, + (CMDLINE_ADDR - REAL_ADDR - SETUP_HEAP_GAP).to_bytes(4, "little"), + "heap_end_ptr", + ), + (CMD_LINE_PTR_OFFSET, CMDLINE_ADDR.to_bytes(4, "little"), "cmd_line_ptr"), + ] + if initrd_size <= 0: + return fields + + if protocol >= XLF_PROTOCOL: + xloadflags = int.from_bytes( + image[XLOADFLAGS_OFFSET : XLOADFLAGS_OFFSET + 2], "little" + ) + initrd_max = ( + 0xFFFFFFFF + if xloadflags & XLF_CAN_BE_LOADED_ABOVE_4G + else INITRD_ADDR_MAX_DEFAULT + ) + elif protocol >= IAM_PROTOCOL: + declared = int.from_bytes( + image[INITRD_ADDR_MAX_OFFSET : INITRD_ADDR_MAX_OFFSET + 4], "little" + ) + initrd_max = declared or INITRD_ADDR_MAX_DEFAULT + else: + initrd_max = INITRD_ADDR_MAX_DEFAULT + available = LOW_MEMORY_SPLIT - ACPI_DATA_SIZE + if initrd_max >= available: + initrd_max = available - 1 + if initrd_size >= initrd_max: + raise ValueError( + f"an initrd of {initrd_size} bytes does not fit below 0x{initrd_max:x}" ) - image[LOADFLAGS_OFFSET] = loadflags & ~CAN_USE_HEAP + initrd_addr = (initrd_max - initrd_size) & ~(INITRD_ALIGNMENT - 1) + fields.append( + (RAMDISK_IMAGE_OFFSET, initrd_addr.to_bytes(4, "little"), "ramdisk_image") + ) + fields.append( + (RAMDISK_SIZE_OFFSET, initrd_size.to_bytes(4, "little"), "ramdisk_size") + ) + return fields + +def normalize(image: bytearray, initrd_size: int) -> list: + """Write the boot-loader-written fields. Returns the fields it changed.""" + changed = [] + for offset, value, name in target_fields(bytes(image), initrd_size): + old = bytes(image[offset : offset + len(value)]) + if old != value: + changed.append((name, offset, old.hex(), value.hex())) + image[offset : offset + len(value)] = value return changed @@ -97,6 +183,10 @@ def main() -> int: """Normalize the image named on the command line, or check it in place.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("bzimage", help="kernel image to normalize in place") + parser.add_argument( + "initrd", + help="initrd the image ships, whose size decides the initrd address", + ) parser.add_argument( "--check", action="store_true", @@ -106,8 +196,11 @@ def main() -> int: with open(args.bzimage, "rb") as f: image = bytearray(f.read()) + with open(args.initrd, "rb") as f: + f.seek(0, 2) + initrd_size = f.tell() - changed = normalize(image) + changed = normalize(image, initrd_size) for name, offset, old, new in changed: print(f"{args.bzimage}: {name} (0x{offset:03x}) {old} -> {new}") diff --git a/os/mkosi/build.sh b/os/mkosi/build.sh index 63f567b5a..8d13cfed4 100755 --- a/os/mkosi/build.sh +++ b/os/mkosi/build.sh @@ -208,4 +208,6 @@ build_one "$BUILD_DIR/b" prod "$jobs_b" cmp "$BUILD_DIR/a/dstack-$DSTACK_VERSION.tar.gz" "$BUILD_DIR/b/dstack-$DSTACK_VERSION.tar.gz" cmp "$BUILD_DIR/a/dstack-$DSTACK_VERSION-uki.tar.gz" \ "$BUILD_DIR/b/dstack-$DSTACK_VERSION-uki.tar.gz" +cmp "$BUILD_DIR/a/dstack-$DSTACK_VERSION-kernel-devel.tar.gz" \ + "$BUILD_DIR/b/dstack-$DSTACK_VERSION-kernel-devel.tar.gz" echo 'reproducibility check passed' diff --git a/os/mkosi/components/dstack-rust/dstack-rust-build.sh b/os/mkosi/components/dstack-rust/dstack-rust-build.sh index 32b167edc..4d3a66e74 100755 --- a/os/mkosi/components/dstack-rust/dstack-rust-build.sh +++ b/os/mkosi/components/dstack-rust/dstack-rust-build.sh @@ -12,6 +12,10 @@ install -d "$DEST/usr/bin" "$DEST/usr/lib/systemd/system" \ for s in dstack-prepare ephemeral-docker app-compose; do install -m0755 "$ROOT/os/common/rootfs/$s.sh" "$DEST/usr/bin/$s.sh" done +# Experimental helper, kept out of PATH so it is not mistaken for a stable +# command; see docs/tdx-guest-scaling.md. +install -D -m0755 "$ROOT/os/common/rootfs/tdx-guest-tune.sh" \ + "$DEST/usr/lib/dstack/tdx-guest-tune.sh" install -m0644 "$ROOT/os/common/rootfs/"*.service \ "$ROOT/os/common/rootfs/dstack-guest-agent.socket" "$DEST/usr/lib/systemd/system/" install -m0644 "$ROOT/os/common/rootfs/journald.conf" "$DEST/etc/systemd/journald.conf.d/dstack.conf" diff --git a/os/mkosi/components/kernel/kernel-build.sh b/os/mkosi/components/kernel/kernel-build.sh index f6ea91c31..48c986e97 100755 --- a/os/mkosi/components/kernel/kernel-build.sh +++ b/os/mkosi/components/kernel/kernel-build.sh @@ -7,8 +7,10 @@ MKOSI_DIR="$ROOT/os/mkosi" source "$MKOSI_DIR/versions.env" BUILD_DIR=${1:?build directory required} STAGING=${2:?staging tree required} +DEVEL_STAGING=${3:?kernel-devel staging tree required} BUILD_DIR=$(realpath -m "$BUILD_DIR") STAGING=$(realpath -m "$STAGING") +DEVEL_STAGING=$(realpath -m "$DEVEL_STAGING") JOBS=${JOBS:-$(nproc)} export KBUILD_BUILD_TIMESTAMP="@${SOURCE_DATE_EPOCH:?SOURCE_DATE_EPOCH required}" export KBUILD_BUILD_USER=dstack KBUILD_BUILD_HOST=reproducible @@ -21,7 +23,7 @@ export KCFLAGS="${KCFLAGS:-} $kernel_map" export KAFLAGS="${KAFLAGS:-} $kernel_map" export KCPPFLAGS="${KCPPFLAGS:-} $kernel_map" -mkdir -p "$BUILD_DIR/downloads" "$BUILD_DIR/kernel-build" "$STAGING" +mkdir -p "$BUILD_DIR/downloads" "$BUILD_DIR/kernel-build" "$STAGING" "$DEVEL_STAGING" # pahole 1.25 produces a different BTF type order when its encoder runs with # Kbuild's parallel job count. Keep compilation parallel, but serialize BTF # encoding so vmlinux and bzImage are byte-for-byte reproducible. @@ -48,7 +50,8 @@ echo "$KERNEL_SHA256 $tarball" | sha256sum --check --status || { echo "kernel source checksum mismatch" >&2; exit 1; } src="$BUILD_DIR/linux-$KERNEL_VERSION" -rm -rf "$src" "$BUILD_DIR/kernel-build" "$STAGING/usr/lib/modules/$KERNEL_VERSION-dstack" +rm -rf "$src" "$BUILD_DIR/kernel-build" "$STAGING/usr/lib/modules/$KERNEL_VERSION-dstack" \ + "$DEVEL_STAGING/usr/lib/dstack/kernel-devel" tar -C "$BUILD_DIR" --no-same-owner -xf "$tarball" for patch in \ "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0001-x86-tdx-select-dma-direct-remap.patch" \ @@ -56,7 +59,11 @@ for patch in \ "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0003-dma-direct-return-struct-page-from-alloc-from-pool.patch" \ "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0004-dma-pool-free-atomic-pool-pages-by-physical-address.patch" \ "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0005-swiotlb-preserve-allocation-virtual-address.patch" \ - "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0006-swiotlb-free-dynamic-pools-from-process-context.patch"; do + "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0006-swiotlb-free-dynamic-pools-from-process-context.patch" \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0007-sched-smp-batch-wake-q-ipis-in-tdx-guests.patch" \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0008-x86-kvm-runtime-pv-single-target-ipi-in-tdx-guests.patch" \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0009-mm-page-alloc-keep-accepted-reserve-before-watermarks.patch" \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0010-x86-tdx-convert-swiotlb-to-shared-after-smp-bringup.patch"; do patch -d "$src" -p1 --fuzz=0 < "$patch" done @@ -83,4 +90,16 @@ install -Dm0644 "$BUILD_DIR/kernel-build/arch/x86/boot/bzImage" \ "$STAGING/usr/lib/modules/$KERNEL_VERSION-dstack/vmlinuz" install -Dm0644 "$BUILD_DIR/kernel-build/.config" \ "$STAGING/usr/lib/modules/$KERNEL_VERSION-dstack/config" -find "$STAGING" -print0 | xargs -0r touch --no-dereference --date="@$SOURCE_DATE_EPOCH" +# Export the external-module build tree as a release artifact, using the same +# upstream helper `make bindeb-pkg` uses for linux-headers. It is staged into +# the rootfs install tree and moved out by mkosi.finalize before measurement. +devel="$DEVEL_STAGING/usr/lib/dstack/kernel-devel" +headers="$devel/linux-headers-$(cat "$BUILD_DIR/kernel-build/include/config/kernel.release")" +make -C "$src" O="$BUILD_DIR/kernel-build" run-command \ + KBUILD_RUN_COMMAND="$src/scripts/package/install-extmod-build $headers" +install -m0644 "$BUILD_DIR/kernel-build/.config" "$headers/.config" +# A -fverbose-asm intermediate that records the absolute build path, which no +# prefix map rewrites. Only its generated .h is needed. +rm -f "$headers/scripts/mod/devicetable-offsets.s" +find "$STAGING" "$DEVEL_STAGING" -print0 | \ + xargs -0r touch --no-dereference --date="@$SOURCE_DATE_EPOCH" diff --git a/os/mkosi/components/kernel/kernel.config b/os/mkosi/components/kernel/kernel.config index 822e539b9..f21a9ea2e 100644 --- a/os/mkosi/components/kernel/kernel.config +++ b/os/mkosi/components/kernel/kernel.config @@ -15,6 +15,11 @@ CONFIG_ACPI_TABLE_UPGRADE=y CONFIG_HYPERVISOR_GUEST=y CONFIG_PARAVIRT=y CONFIG_KVM_GUEST=y +# Guest halt polling is built as a module so it can be enabled and disabled at +# runtime (modprobe/rmmod cpuidle_haltpoll). Unloading restores the default +# idle path and host halt polling; a built-in driver can only bind at boot. +CONFIG_CPU_IDLE_GOV_HALTPOLL=y +CONFIG_HALTPOLL_CPUIDLE=m CONFIG_INTEL_TDX_GUEST=y CONFIG_TDX_GUEST_DRIVER=y CONFIG_TSM_REPORTS=y diff --git a/os/mkosi/components/kernel/kernel.sh b/os/mkosi/components/kernel/kernel.sh index 62817415b..9217b66b0 100644 --- a/os/mkosi/components/kernel/kernel.sh +++ b/os/mkosi/components/kernel/kernel.sh @@ -1,8 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # shellcheck shell=bash disable=SC2034 COMPONENT_NAME=kernel -COMPONENT_CACHE_PATHS=("linux-$KERNEL_VERSION" kernel-build component-stages/kernel) -COMPONENT_ROOTFS_TREES=() +COMPONENT_CACHE_PATHS=("linux-$KERNEL_VERSION" kernel-build component-stages/kernel + component-stages/kernel-devel) +# Staged via the rootfs tree; mkosi.finalize moves it out before measurement. +COMPONENT_ROOTFS_TREES=(component-stages/kernel-devel) COMPONENT_KERNEL_TREES=(component-stages/kernel) component_cache_key() { @@ -13,11 +15,16 @@ component_cache_key() { "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0003-dma-direct-return-struct-page-from-alloc-from-pool.patch" \ "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0004-dma-pool-free-atomic-pool-pages-by-physical-address.patch" \ "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0005-swiotlb-preserve-allocation-virtual-address.patch" \ - "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0006-swiotlb-free-dynamic-pools-from-process-context.patch" + "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0006-swiotlb-free-dynamic-pools-from-process-context.patch" \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0007-sched-smp-batch-wake-q-ipis-in-tdx-guests.patch" \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0008-x86-kvm-runtime-pv-single-target-ipi-in-tdx-guests.patch" \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0009-mm-page-alloc-keep-accepted-reserve-before-watermarks.patch" \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0010-x86-tdx-convert-swiotlb-to-shared-after-smp-bringup.patch" key_tools gcc ld make pahole key_packages binutils dwarves bc bison flex libssl-dev libelf-dev } component_build() { - "$COMPONENT_PATH/kernel-build.sh" "$WORK" "$WORK/component-stages/kernel" + "$COMPONENT_PATH/kernel-build.sh" "$WORK" "$WORK/component-stages/kernel" \ + "$WORK/component-stages/kernel-devel" } diff --git a/os/mkosi/mkosi.finalize b/os/mkosi/mkosi.finalize index 10c7e50d0..38032a15c 100755 --- a/os/mkosi/mkosi.finalize +++ b/os/mkosi/mkosi.finalize @@ -20,6 +20,8 @@ cp -a "$TREE/usr/lib/modules" "$KSTAGE/usr/lib/" install -m0644 "$TREE/usr/lib/dstack/firmware/ovmf.fd" "$KSTAGE/ovmf.fd" install -m0644 "$TREE/usr/lib/dstack/firmware/ovmf-sev.fd" "$KSTAGE/ovmf-sev.fd" install -m0755 "$TREE/usr/lib/dstack/image-tools/"* "$OUTPUTDIR/.image-tools/" +# The kernel build tree is a release artifact, never part of the measured rootfs. +mv "$TREE/usr/lib/dstack/kernel-devel" "$KSTAGE/kernel-devel" rm -rf "$TREE/usr/lib/dstack/firmware" "$TREE/usr/lib/dstack/image-tools" rm -f "$TREE/var/cache/ldconfig/aux-cache" diff --git a/os/mkosi/parity.json b/os/mkosi/parity.json index 0b10d4802..2b2a627ef 100644 --- a/os/mkosi/parity.json +++ b/os/mkosi/parity.json @@ -153,6 +153,7 @@ "CONFIG_NETFILTER_XT_MATCH_COMMENT=m" ], "prod_forbidden_paths": [ + "usr/lib/dstack/kernel-devel", "usr/bin/dstack-tee-simulator", "usr/bin/swtpm", "usr/lib/systemd/system/dstack-prepare.service.d/tee-simulator.conf", diff --git a/os/mkosi/scripts/make-release-artifacts.sh b/os/mkosi/scripts/make-release-artifacts.sh index 8752e8069..c5971da38 100755 --- a/os/mkosi/scripts/make-release-artifacts.sh +++ b/os/mkosi/scripts/make-release-artifacts.sh @@ -124,16 +124,31 @@ touch -d "@$SOURCE_DATE_EPOCH" "$OUT/files/"* is_dev=false; name=dstack if [[ $FLAVOR == dev ]]; then is_dev=true; name=dstack-dev; fi + +# The kernel build tree is archived beside, not inside, the measured payload. +kernel_devel_artifact= +if [[ ${DSTACK_TAR_RELEASE:-1} == 1 ]]; then + tar -C "$KERNEL_TREE" --sort=name --mtime="@$SOURCE_DATE_EPOCH" \ + --owner=0 --group=0 --numeric-owner \ + --transform="s|^kernel-devel|$name-$DSTACK_VERSION-kernel-devel|" \ + -czf "$OUT/files/kernel-devel.tar.gz" kernel-devel + touch -d "@$SOURCE_DATE_EPOCH" "$OUT/files/kernel-devel.tar.gz" + kernel_devel_artifact=files/kernel-devel.tar.gz +fi + python3 - "$OUT/artifact-manifest.json" "$name" "$FLAVOR" "$is_dev" \ - "${DSTACK_SOURCE_REVISION:?}" "$root_hash" "$data_size" "$DSTACK_VERSION" "$OVMF_VARIANT" <<'PY' + "${DSTACK_SOURCE_REVISION:?}" "$root_hash" "$data_size" "$DSTACK_VERSION" "$OVMF_VARIANT" \ + "$kernel_devel_artifact" <<'PY' import json, sys -path,name,flavor,is_dev,revision,root_hash,data_size,version,ovmf_variant=sys.argv[1:] +(path,name,flavor,is_dev,revision,root_hash,data_size,version,ovmf_variant, + kernel_devel)=sys.argv[1:] data={"schema_version":1,"backend":"mkosi","image":{"name":name,"version":version, "flavor":flavor,"is_dev":is_dev=="true"},"source":{"git_revision":revision}, "boot":{"ovmf_variant":ovmf_variant},"verity":{"root_hash":root_hash,"data_size":data_size}, "artifacts":{"initramfs":"files/initramfs.cpio.gz","kernel":"files/bzImage", "firmware":"files/ovmf.fd","rootfs_verity":"files/rootfs.squashfs.verity", -"firmware_sev":"files/ovmf-sev.fd","uki":"files/dstack-uki.efi"}, +"firmware_sev":"files/ovmf-sev.fd","uki":"files/dstack-uki.efi", +"kernel_devel":kernel_devel or None}, "backend_metadata":{"machine":"dstack","distribution":"debian"}} with open(path,"w") as f: json.dump(data,f,indent=2); f.write("\n") PY diff --git a/os/spec/artifact-manifest.schema.json b/os/spec/artifact-manifest.schema.json index 44fe0ab11..1a75eadc6 100644 --- a/os/spec/artifact-manifest.schema.json +++ b/os/spec/artifact-manifest.schema.json @@ -91,6 +91,9 @@ }, "uki": { "oneOf": [{"$ref": "#/$defs/artifactPath"}, {"type": "null"}] + }, + "kernel_devel": { + "oneOf": [{"$ref": "#/$defs/artifactPath"}, {"type": "null"}] } } }, diff --git a/os/tests/test-kernel-header-normalization.sh b/os/tests/test-kernel-header-normalization.sh index 73bb9e710..b5f959a6c 100755 --- a/os/tests/test-kernel-header-normalization.sh +++ b/os/tests/test-kernel-header-normalization.sh @@ -11,9 +11,11 @@ # 0007-OvmfPkg-...-normalize-setup-header.patch (firmware) # # If they ever disagree, every CVM fails attestation with an RTMR[1] mismatch -# and nothing else points at why. So this test parses the field table out of -# both and compares them, then exercises the Python side against a synthetic -# bzImage. +# and nothing else points at why. So this test checks that both write the same +# fields with the same constants, then exercises the Python side against a +# synthetic bzImage. dstack-mr is the third implementation -- it predicts this +# layout for an image that does not declare the flag -- so its constants are +# checked too. set -euo pipefail here=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) @@ -24,50 +26,46 @@ patch=$root/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/000 [ -x "$script" ] || { echo "missing $script" >&2; exit 1; } [ -f "$patch" ] || { echo "missing $patch" >&2; exit 1; } -python3 - "$script" "$patch" <<'PYEOF' +kernel_rs=$root/dstack/dstack-mr/src/kernel.rs +machine_rs=$root/dstack/dstack-mr/src/machine.rs +[ -f "$kernel_rs" ] || { echo "missing $kernel_rs" >&2; exit 1; } +[ -f "$machine_rs" ] || { echo "missing $machine_rs" >&2; exit 1; } + +python3 - "$script" "$patch" "$kernel_rs" "$machine_rs" <<'PYEOF' import importlib.util import re import sys -script_path, patch_path = sys.argv[1], sys.argv[2] +script_path, patch_path, kernel_rs_path, machine_rs_path = sys.argv[1:5] + +# Importing the script must not leave a .pyc beside it: a cached one whose +# source changed within the same second, without changing size, is reused, and +# this test would then check bytecode that is no longer on disk. +sys.dont_write_bytecode = True spec = importlib.util.spec_from_file_location("normalize", script_path) normalize = importlib.util.module_from_spec(spec) spec.loader.exec_module(normalize) -py_fields = sorted((offset, size) for offset, size, _ in normalize.WRITE_FIELDS) +patch_text = open(patch_path, encoding="utf-8").read() +machine_rs = open(machine_rs_path, encoding="utf-8").read() +# Only what dstack-mr compiles into a release counts: a constant that survives +# in `#[cfg(test)]` alone would let the two sides drift while this test passes. +kernel_rs = open(kernel_rs_path, encoding="utf-8").read().split("mod tests {")[0] -# Added lines of the form " { 0x210, 1 }, // type_of_loader". -patch_text = open(patch_path, encoding="ascii").read() -ovmf_fields = sorted( - (int(offset, 16), int(size)) - for offset, size in re.findall( - r"^\+\s*\{\s*(0x[0-9A-Fa-f]+),\s*(\d+)\s*\},", patch_text, re.M - ) -) -assert ovmf_fields, "no field table found in the OVMF patch" -assert py_fields == ovmf_fields, ( - "normalization field tables disagree\n" - f" image build: {[(hex(o), s) for o, s in py_fields]}\n" - f" OVMF patch: {[(hex(o), s) for o, s in ovmf_fields]}" -) +def ovmf(name): + match = re.search(rf"^\+#define {name}\s+(0x[0-9A-Fa-f]+)", patch_text, re.M) + assert match, f"the OVMF patch no longer defines {name}" + return int(match.group(1), 16) -for token in ("LINUX_HDR_MIN_PROTOCOL 0x0209", "LINUX_LOADFLAGS_CAN_USE_HEAP 0x80"): - assert token in patch_text, f"OVMF patch no longer defines {token}" -assert normalize.MIN_PROTOCOL == 0x0209 -assert normalize.CAN_USE_HEAP == 0x80 -# `modify` fields carry kernel-supplied values and must never be normalized. -# code32_start is the protected-mode entry point; zeroing it bricks the kernel. -for forbidden in (0x1F2, 0x1FA, 0x211, 0x212, 0x214): - assert all(offset != forbidden for offset, _ in py_fields), ( - f"0x{forbidden:x} is a `modify` field and must not be zeroed" - ) +INITRD_SIZE = 6_454_798 + # Synthetic bzImage: a plausible built kernel, then the same image with every # boot-loader-written field filled in the way QEMU fills them. -def make_image(patched: bool) -> bytearray: +def make_image(patched: bool, xloadflags: int = 0, initrd_addr: int = 0) -> bytearray: image = bytearray(0x1000) image[0x202:0x206] = b"HdrS" image[0x206:0x208] = (0x020F).to_bytes(2, "little") @@ -77,31 +75,183 @@ def make_image(patched: bool) -> bytearray: image[0x212:0x214] = (0x8000).to_bytes(2, "little") # setup_move_size image[0x214:0x218] = (0x100000).to_bytes(4, "little") # code32_start image[0x224:0x226] = (0x50A0).to_bytes(2, "little") # heap_end_ptr + image[0x236:0x238] = xloadflags.to_bytes(2, "little") if patched: + # What QEMU wrote for some other guest: the initrd address is the only + # field its memory size reaches, so it is the caller's to vary. image[0x210] = 0xB0 image[0x211] |= 0x80 - image[0x218:0x21C] = (0xA97FC000).to_bytes(4, "little") - image[0x21C:0x220] = (0x0062A954).to_bytes(4, "little") + image[0x218:0x21C] = initrd_addr.to_bytes(4, "little") + image[0x21C:0x220] = (INITRD_SIZE).to_bytes(4, "little") image[0x224:0x226] = (0xFE00).to_bytes(2, "little") image[0x228:0x22C] = (0x20000).to_bytes(4, "little") return image -built, patched = make_image(False), make_image(True) + +# Both sides must write the same fields at the same widths. Read the Python +# side out of the script instead of restating it here, so that a field added or +# widened on one side alone fails this test. +py_fields = { + (offset, len(value)) + for offset, value, _ in normalize.target_fields(bytes(make_image(False)), INITRD_SIZE) +} + + +def ovmf_written_fields(): + """The (offset, size) the OVMF patch assigns to, parsed from its code.""" + written = {} + depth = 0 + for line in patch_text.splitlines(): + if not line.startswith("+"): + continue + body = line[1:] + for match in re.finditer(r"Data\[LINUX_HDR_(\w+)_OFFSET\]\s*\|?=", body): + written[match.group(1)] = 1 + if depth: + for match in re.finditer(r"LINUX_HDR_(\w+)_OFFSET", body): + written[match.group(1)] = 4 + if "LinuxHdrWrite32 (" in body: + # A call reaching the closing `);` on its own line spans several. + after = body.split("LinuxHdrWrite32 (", 1)[1] + for match in re.finditer(r"LINUX_HDR_(\w+)_OFFSET", after): + written[match.group(1)] = 4 + depth = 0 if ");" in after else 1 + elif depth and ");" in body: + depth = 0 + return {(ovmf(f"LINUX_HDR_{name}_OFFSET"), size) for name, size in written.items()} + + +ovmf_fields = ovmf_written_fields() +assert py_fields == ovmf_fields, (sorted(py_fields), sorted(ovmf_fields)) +assert len(py_fields) == 6, sorted(py_fields) + +# Every value the layout depends on, in the firmware. +for name, value in ( + ("QEMU_LINUX_TYPE_OF_LOADER", normalize.TYPE_OF_LOADER_QEMU), + ("QEMU_LINUX_REAL_ADDR", normalize.REAL_ADDR), + ("QEMU_LINUX_CMDLINE_ADDR", normalize.CMDLINE_ADDR), + ("QEMU_LINUX_SETUP_HEAP_GAP", normalize.SETUP_HEAP_GAP), + ("QEMU_LOW_MEMORY_SPLIT", normalize.LOW_MEMORY_SPLIT), + ("QEMU_ACPI_DATA_SIZE", normalize.ACPI_DATA_SIZE), + ("LINUX_INITRD_ADDR_MAX_DEFAULT", normalize.INITRD_ADDR_MAX_DEFAULT), + ("QEMU_INITRD_ALIGNMENT", normalize.INITRD_ALIGNMENT), + ("LINUX_XLF_CAN_BE_LOADED_ABOVE_4G", normalize.XLF_CAN_BE_LOADED_ABOVE_4G), + ("LINUX_HDR_MIN_PROTOCOL", normalize.MIN_PROTOCOL), + ("LINUX_LOADFLAGS_CAN_USE_HEAP", normalize.CAN_USE_HEAP), + ("LINUX_LOADFLAGS_LOADED_HIGH", normalize.LOADED_HIGH), +): + assert ovmf(name) == value, f"{name} is {ovmf(name):#x} in the OVMF patch" + +assert normalize.MIN_PROTOCOL == 0x0202 +assert normalize.CAN_USE_HEAP == 0x80 +assert normalize.LOADED_HIGH == 0x01 + +# The same values in dstack-mr, matched where it computes the layout rather +# than anywhere in the file: it is the implementation an earlier release's +# verifier or KMS runs against these images. +for text, spelling, what in ( + (kernel_rs, f"kd[0x210] = {normalize.TYPE_OF_LOADER_QEMU:#x}", "type_of_loader"), + ( + kernel_rs, + f"({normalize.REAL_ADDR:#x}_u32, {normalize.CMDLINE_ADDR:#x}_u32)", + "the real-mode and command-line addresses", + ), + ( + kernel_rs, + f"saturating_sub({normalize.SETUP_HEAP_GAP:#x})", + "the setup-heap gap", + ), + ( + kernel_rs, + f"xlf & {normalize.XLF_CAN_BE_LOADED_ABOVE_4G:#04x}", + "XLF_CAN_BE_LOADED_ABOVE_4G", + ), + ( + kernel_rs, + f"{normalize.INITRD_ADDR_MAX_DEFAULT:#x}", + "the default initrd ceiling", + ), + (kernel_rs, f"& !{normalize.INITRD_ALIGNMENT - 1}", "the initrd alignment"), + ( + kernel_rs, + f"TDX_KERNEL_HASH_COMPAT_2G_MEMORY: u64 = {normalize.LOW_MEMORY_SPLIT:#x}", + "the 2 GiB split", + ), + (machine_rs, f"{normalize.ACPI_DATA_SIZE:#x}", "the ACPI window"), +): + assert spelling in text, f"dstack-mr no longer spells {what} as {spelling}" + +# `modify` fields carry kernel-supplied values and must never be written. +# code32_start is the protected-mode entry point; overwriting it bricks the +# kernel. loadflags is only OR-ed with CAN_USE_HEAP, never replaced. +for forbidden in (0x1F2, 0x1FA, 0x212, 0x214): + assert forbidden not in {offset for offset, _ in py_fields}, ( + f"0x{forbidden:x} is a `modify` field and must not be written" + ) + +built, patched = make_image(False), make_image(True, initrd_addr=0x37BF3000) assert built != patched -normalize.normalize(built) -normalize.normalize(patched) +normalize.normalize(built, INITRD_SIZE) +normalize.normalize(patched, INITRD_SIZE) assert built == patched, "normalizing a QEMU-patched kernel must reproduce the shipped one" -# Idempotent, so OVMF re-running it over an already normalized kernel is a no-op. +# The layout QEMU writes, which is what dstack-mr predicts for an image that +# does not declare the flag. This kernel declares no xloadflags, so QEMU caps +# the initrd at the 0x37ffffff a kernel that cannot be loaded above 4G gets. +assert built[0x210] == 0xB0 +assert built[0x211] == 0x81 +assert built[0x218:0x21C] == (0x379D8000).to_bytes(4, "little") +assert built[0x21C:0x220] == (INITRD_SIZE).to_bytes(4, "little") +assert built[0x224:0x228] == (0xFE00).to_bytes(4, "little") +assert built[0x228:0x22C] == (0x20000).to_bytes(4, "little") + +# The branch every kernel this build ships takes: XLF_CAN_BE_LOADED_ABOVE_4G +# lifts QEMU's ceiling to 4G, and what caps the initrd is then the below-4G +# window it reserves for ACPI tables with RAM split at 2 GiB. +above_4g = make_image(False, xloadflags=normalize.XLF_CAN_BE_LOADED_ABOVE_4G) +normalize.normalize(above_4g, INITRD_SIZE) +assert above_4g[0x218:0x21C] == (0x7F9B0000).to_bytes(4, "little"), ( + f"initrd address is {int.from_bytes(above_4g[0x218:0x21C], 'little'):#x}" +) +assert above_4g[0x21C:0x220] == (INITRD_SIZE).to_bytes(4, "little") + +# 0x40 is XLF_5LEVEL_ENABLED, not XLF_CAN_BE_LOADED_ABOVE_4G. Reading it as the +# latter -- which all three implementations once did -- moves the initrd about +# 1.1 GiB away from where QEMU puts it, for any kernel built without 5-level +# paging. +five_level = make_image(False, xloadflags=0x0040) +normalize.normalize(five_level, INITRD_SIZE) +assert five_level[0x218:0x21C] == built[0x218:0x21C], ( + "the 5-level flag must not raise the initrd ceiling" +) + +# A kernel loaded low takes QEMU's other command-line address, which depends on +# the command line the host passes: there is nothing to normalize to, and the +# build has to say so rather than write a value QEMU never writes. +low = make_image(False) +low[0x211] &= ~normalize.LOADED_HIGH +try: + normalize.normalize(low, INITRD_SIZE) +except ValueError: + pass +else: + raise AssertionError("the image build must reject a kernel not loaded high") +assert "kernel is not loaded high" in patch_text, ( + "the OVMF side must leave a low-loaded kernel as served" +) + +# Independent of guest RAM: an initrd address QEMU derived for another guest +# size normalizes to the same bytes, which is what makes the host's QEMU +# version and memory size irrelevant. again = bytearray(built) -assert normalize.normalize(again) == [] +assert normalize.normalize(again, INITRD_SIZE) == [] assert again == built -# The `modify` fields survived. +# The `modify` fields survived, and loadflags kept LOADED_HIGH. assert built[0x1F2:0x1F4] == (0x0001).to_bytes(2, "little") assert built[0x1FA:0x1FC] == (0xFFFF).to_bytes(2, "little") -assert built[0x211] == 0x01 +assert built[0x211] & 0x01 assert built[0x212:0x214] == (0x8000).to_bytes(2, "little") assert built[0x214:0x218] == (0x100000).to_bytes(4, "little") @@ -112,7 +262,7 @@ assert built[0x214:0x218] == (0x100000).to_bytes(4, "little") other = bytearray(0x1000) other[0x210] = 0xB0 try: - normalize.normalize(other) + normalize.normalize(other, INITRD_SIZE) except ValueError: pass else: @@ -125,13 +275,13 @@ assert "return;" in patch_text.split('CompareMem (Data + LINUX_HDR_MAGIC_OFFSET' # normalizes. The image build still fails loudly about it. ancient = bytearray(0x1000) ancient[0x202:0x206] = b"HdrS" -ancient[0x206:0x208] = (0x0208).to_bytes(2, "little") +ancient[0x206:0x208] = (0x0201).to_bytes(2, "little") try: - normalize.normalize(ancient) + normalize.normalize(ancient, INITRD_SIZE) except ValueError: pass else: - raise AssertionError("the image build must reject boot protocols below 2.09") + raise AssertionError("the image build must reject boot protocols below 2.02") print("kernel setup-header normalization: OK") PYEOF diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 63757007e..c2beca373 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -36,11 +36,17 @@ EXTRA_CARGO_FLAGS = "-p dstack-guest-agent -p dstack-util -p dstack-volume" inherit cargo_bin +# The staged source lives outside WORKDIR and carries the host checkout's +# ownership. WORKDIR staging has no use for it, and applying it fails outright +# under container / user-namespace builds where the source uid/gid is not +# settable (rsync aborts with "chown/chgrp ... Invalid argument", exit 23, and +# leaves the partially-attributed temp file behind). Drop owner and group, keep +# the rest of archive mode -- permissions still matter for executable bits. do_unpack() { install -d "${S}" "${DSTACK_ROOTFS_FILES}" - rsync -a --exclude=".git" --exclude=".worktrees" --exclude="target" \ + rsync -a --no-owner --no-group --exclude=".git" --exclude=".worktrees" --exclude="target" \ "${DSTACK_CORE_SRC}/" "${S}/" - rsync -a "${DSTACK_ROOTFS_SRC}/" "${DSTACK_ROOTFS_FILES}/" + rsync -a --no-owner --no-group "${DSTACK_ROOTFS_SRC}/" "${DSTACK_ROOTFS_FILES}/" } do_unpack[cleandirs] = "${UNPACKDIR}/repo" diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0007-OvmfPkg-QemuKernelLoaderFsDxe-normalize-setup-header.patch b/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0007-OvmfPkg-QemuKernelLoaderFsDxe-normalize-setup-header.patch index 30f41be5b..098002ed5 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0007-OvmfPkg-QemuKernelLoaderFsDxe-normalize-setup-header.patch +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0007-OvmfPkg-QemuKernelLoaderFsDxe-normalize-setup-header.patch @@ -12,75 +12,129 @@ guests, so from 10.2 on the same kernel measures differently than it does on 10.1 and earlier. Attestation has to know the host's QEMU version to predict its own RTMR[1], and the host declares that version. -Zero the fields the boot protocol says a boot loader owns, which makes the -published bytes independent of what QEMU did. The dstack image build applies -the same normalization to the kernel it ships, so the measurement is the plain -Authenticode hash of that file on every QEMU version. +Write the fields the boot protocol says a boot loader owns, with the values +QEMU <= 10.1 writes, which makes the published bytes independent of what QEMU +did. The dstack image build writes the same values into the kernel it ships, so +the measurement is the plain Authenticode hash of that file on every QEMU +version. -The field set is taken from Documentation/arch/x86/boot.rst rather than from -QEMU's behavior: fields typed `write` are ones the boot loader fills in and the -kernel supplies no value for. Fields typed `modify` are left alone because they -carry real kernel-supplied values. +The values follow from the kernel file and the initrd size -- never from the +QEMU version, and never from guest RAM: the initrd address QEMU derives from +RAM is fixed at the value it resolves to for a guest with 2 GiB or more below +4G. A kernel loaded low is left alone, because QEMU's command-line address on +that path depends on the command line the host passes. + +This writes one fixed header, not a reimplementation of QEMU's loader. Zeros +would serve the measurement goal equally well; these particular values are +chosen because dstack-mr already computes them (patch_kernel in +dstack/dstack-mr/src/kernel.rs) for an image that does not declare +kernel_header_normalized, so a verifier or KMS from an earlier dstack release +can still measure these images. + +Every field written is one QEMU fills in as boot loader. All but one are typed +`write` in Documentation/arch/x86/boot.rst: the boot loader supplies them and +the kernel has no value there. The exception is loadflags, typed `modify +(obligatory)`, of which only CAN_USE_HEAP is set -- a bit the protocol assigns +to the boot loader. Fields carrying real kernel-supplied values are left alone, +and so are the `write` fields QEMU never touches. Booting is unaffected. On the EFI-stub path the real-mode setup code never -runs, and heap_end_ptr -- the only one of these fields a built kernel leaves -non-zero -- is read only when the boot loader has set CAN_USE_HEAP, which this -also clears. +runs, the stub takes the initrd through LoadFile2 and the command line through +its load options, and these are the values QEMU itself wrote for every release +before 10.2. Signed-off-by: dstack --- --- a/OvmfPkg/QemuKernelLoaderFsDxe/QemuKernelLoaderFsDxe.c +++ b/OvmfPkg/QemuKernelLoaderFsDxe/QemuKernelLoaderFsDxe.c -@@ -49,6 +49,111 @@ +@@ -49,6 +49,221 @@ struct KERNEL_BLOB { KERNEL_BLOB *Next; }; +// -+// Offsets into the Linux x86 setup header, from ++// Linux x86 setup-header offsets and flags, from +// Documentation/arch/x86/boot.rst. +// -+#define LINUX_HDR_MAGIC_OFFSET 0x202 -+#define LINUX_HDR_VERSION_OFFSET 0x206 -+#define LINUX_HDR_LOADFLAGS_OFFSET 0x211 -+#define LINUX_HDR_END_OFFSET 0x258 -+ ++#define LINUX_HDR_MAGIC_OFFSET 0x202 ++#define LINUX_HDR_VERSION_OFFSET 0x206 ++#define LINUX_HDR_TYPE_OF_LOADER_OFFSET 0x210 ++#define LINUX_HDR_LOADFLAGS_OFFSET 0x211 ++#define LINUX_HDR_RAMDISK_IMAGE_OFFSET 0x218 ++#define LINUX_HDR_RAMDISK_SIZE_OFFSET 0x21C ++#define LINUX_HDR_HEAP_END_PTR_OFFSET 0x224 ++#define LINUX_HDR_CMD_LINE_PTR_OFFSET 0x228 ++#define LINUX_HDR_INITRD_ADDR_MAX_OFFSET 0x22C ++#define LINUX_HDR_XLOADFLAGS_OFFSET 0x236 ++#define LINUX_HDR_END_OFFSET 0x258 +// -+// CAN_USE_HEAP in loadflags. The kernel builds it clear; a boot loader sets it -+// when it has also filled in heap_end_ptr. ++// xloadflags bit 1. Bit 6, 0x40, is XLF_5LEVEL_ENABLED, not this flag. +// -+#define LINUX_LOADFLAGS_CAN_USE_HEAP 0x80 ++#define LINUX_LOADFLAGS_LOADED_HIGH 0x01 ++#define LINUX_LOADFLAGS_CAN_USE_HEAP 0x80 ++#define LINUX_XLF_CAN_BE_LOADED_ABOVE_4G 0x02 + +// -+// setup_data (0x250) arrived in boot protocol 2.09. Below that the field -+// layout normalized here is not guaranteed, so the image is left alone. ++// cmd_line_ptr moved into the setup header in boot protocol 2.02, ++// initrd_addr_max arrived in 2.03 and xloadflags in 2.12. +// -+#define LINUX_HDR_MIN_PROTOCOL 0x0209 ++#define LINUX_HDR_MIN_PROTOCOL 0x0202 ++#define LINUX_HDR_IAM_PROTOCOL 0x0203 ++#define LINUX_HDR_XLF_PROTOCOL 0x020C + +// -+// Every setup-header field boot.rst types as `write`: the boot loader fills it -+// in and the kernel supplies no value, so zeroing it discards nothing. Fields -+// typed `modify` carry real kernel-supplied values -- code32_start (0x214) is -+// the protected-mode entry point -- and are deliberately absent. ++// What QEMU writes as boot loader: "Qemu" version 0, the real-mode block and ++// command line it loads a kernel at, the 0x200-byte gap it leaves below the ++// command line for the setup heap, the below-4G window it reserves for ACPI ++// tables with RAM split at 2 GiB, the initrd ceiling a kernel that declares ++// none gets, and the alignment it rounds the initrd address down to. Same ++// values as os/image/normalize-kernel-header.py and dstack-mr's patch_kernel. +// -+STATIC CONST struct { -+ UINT16 Offset; -+ UINT16 Size; -+} mLinuxBootLoaderWrittenFields[] = { -+ { 0x210, 1 }, // type_of_loader -+ { 0x218, 4 }, // ramdisk_image -+ { 0x21C, 4 }, // ramdisk_size -+ { 0x224, 2 }, // heap_end_ptr -+ { 0x226, 1 }, // ext_loader_ver -+ { 0x227, 1 }, // ext_loader_type -+ { 0x228, 4 }, // cmd_line_ptr -+ { 0x23C, 4 }, // hardware_subarch -+ { 0x240, 8 }, // hardware_subarch_data -+ { 0x250, 8 }, // setup_data -+}; ++#define QEMU_LINUX_TYPE_OF_LOADER 0xB0 ++#define QEMU_LINUX_REAL_ADDR 0x10000 ++#define QEMU_LINUX_CMDLINE_ADDR 0x20000 ++#define QEMU_LINUX_SETUP_HEAP_GAP 0x200 ++#define QEMU_LOW_MEMORY_SPLIT 0x80000000 ++#define QEMU_ACPI_DATA_SIZE 0x28000 ++#define LINUX_INITRD_ADDR_MAX_DEFAULT 0x37FFFFFF ++#define QEMU_INITRD_ALIGNMENT 0x1000 ++ ++STATIC ++UINT32 ++LinuxHdrRead ( ++ IN CONST UINT8 *Data, ++ IN UINTN Offset, ++ IN UINTN Size ++ ) ++{ ++ UINT32 Value; ++ UINTN Index; ++ ++ Value = 0; ++ for (Index = 0; Index < Size; Index++) { ++ Value |= (UINT32)Data[Offset + Index] << (8 * Index); ++ } ++ ++ return Value; ++} ++ ++STATIC ++VOID ++LinuxHdrWrite32 ( ++ IN OUT UINT8 *Data, ++ IN UINTN Offset, ++ IN UINT32 Value ++ ) ++{ ++ UINTN Index; ++ ++ for (Index = 0; Index < 4; Index++) { ++ Data[Offset + Index] = (UINT8)(Value >> (8 * Index)); ++ } ++} + +/** -+ Clear the setup-header fields a boot loader owns, so the measured kernel does -+ not depend on what QEMU wrote there. ++ Normalize the setup-header fields a boot loader owns, so the measured kernel ++ does not depend on what QEMU wrote there. + + QEMU acts as the boot loader for -kernel and fills these fields in before + serving the image over fw_cfg, which changes the bytes this driver publishes @@ -88,9 +142,13 @@ Signed-off-by: dstack + doing that for confidential guests (commit a7542a38f399), so without this the + same kernel measures differently depending on the host's QEMU. + -+ Zeroing is idempotent and matches what the image build already writes into -+ the shipped kernel, so the measurement equals the Authenticode hash of the -+ file on disk on every QEMU version. ++ The values written are the ones QEMU <= 10.1 writes, for a kernel loaded ++ high in a guest with 2 GiB or more of RAM below 4G: they follow from the kernel file and the initrd ++ size, never from the QEMU version or from guest RAM. The dstack image build ++ writes the same values into the kernel it ships, so the measurement is the ++ Authenticode hash of that file on every QEMU version, and dstack-mr computes ++ them when it predicts RTMR[1] for an image that does not declare ++ kernel_header_normalized. Writing them is idempotent. + + @param[in,out] Data The kernel blob, starting at the setup header. + @param[in] Size Size of the blob in bytes. @@ -102,8 +160,10 @@ Signed-off-by: dstack + IN UINT32 Size + ) +{ -+ UINTN Idx; -+ UINT16 Protocol; ++ UINT32 Protocol; ++ UINT32 InitrdSize; ++ UINT32 InitrdMax; ++ UINT32 AvailableMem; + + if (Size < LINUX_HDR_END_OFFSET) { + return; @@ -117,8 +177,7 @@ Signed-off-by: dstack + return; + } + -+ Protocol = (UINT16)(Data[LINUX_HDR_VERSION_OFFSET] | -+ (Data[LINUX_HDR_VERSION_OFFSET + 1] << 8)); ++ Protocol = LinuxHdrRead (Data, LINUX_HDR_VERSION_OFFSET, 2); + if (Protocol < LINUX_HDR_MIN_PROTOCOL) { + DEBUG (( + DEBUG_WARN, @@ -130,20 +189,87 @@ Signed-off-by: dstack + return; + } + -+ for (Idx = 0; Idx < ARRAY_SIZE (mLinuxBootLoaderWrittenFields); Idx++) { -+ ZeroMem ( -+ Data + mLinuxBootLoaderWrittenFields[Idx].Offset, -+ mLinuxBootLoaderWrittenFields[Idx].Size -+ ); ++ if ((Data[LINUX_HDR_LOADFLAGS_OFFSET] & LINUX_LOADFLAGS_LOADED_HIGH) == 0) { ++ // ++ // A kernel loaded low takes QEMU's other command-line address, ++ // 0x9a000 - cmdline_size, which depends on the command line the host ++ // passes. There is no single value to normalize to, so leave the header ++ // alone. The dstack image build refuses to ship such a kernel. ++ // ++ DEBUG (( ++ DEBUG_WARN, ++ "%a: kernel is not loaded high; setup header left as served\n", ++ __func__ ++ )); ++ return; ++ } ++ ++ Data[LINUX_HDR_TYPE_OF_LOADER_OFFSET] = QEMU_LINUX_TYPE_OF_LOADER; ++ Data[LINUX_HDR_LOADFLAGS_OFFSET] |= LINUX_LOADFLAGS_CAN_USE_HEAP; ++ // ++ // heap_end_ptr is two bytes, but the value never exceeds 16 bits and the two ++ // bytes above it (ext_loader_ver, ext_loader_type) are zero in a built kernel ++ // and untouched by QEMU, so writing the word zero-extended to 32 bits is what ++ // the image build and dstack-mr also write. ++ // ++ LinuxHdrWrite32 ( ++ Data, ++ LINUX_HDR_HEAP_END_PTR_OFFSET, ++ QEMU_LINUX_CMDLINE_ADDR - QEMU_LINUX_REAL_ADDR - QEMU_LINUX_SETUP_HEAP_GAP ++ ); ++ LinuxHdrWrite32 (Data, LINUX_HDR_CMD_LINE_PTR_OFFSET, QEMU_LINUX_CMDLINE_ADDR); ++ ++ QemuFwCfgSelectItem (QemuFwCfgItemInitrdSize); ++ InitrdSize = QemuFwCfgRead32 (); ++ if (InitrdSize == 0) { ++ // ++ // No initrd: QEMU leaves ramdisk_image and ramdisk_size alone, and a built ++ // kernel ships them zero. ++ // ++ return; ++ } ++ ++ if (Protocol >= LINUX_HDR_XLF_PROTOCOL) { ++ InitrdMax = (LinuxHdrRead (Data, LINUX_HDR_XLOADFLAGS_OFFSET, 2) & ++ LINUX_XLF_CAN_BE_LOADED_ABOVE_4G) != 0 ? ++ MAX_UINT32 : LINUX_INITRD_ADDR_MAX_DEFAULT; ++ } else if (Protocol >= LINUX_HDR_IAM_PROTOCOL) { ++ InitrdMax = LinuxHdrRead (Data, LINUX_HDR_INITRD_ADDR_MAX_OFFSET, 4); ++ if (InitrdMax == 0) { ++ InitrdMax = LINUX_INITRD_ADDR_MAX_DEFAULT; ++ } ++ } else { ++ InitrdMax = LINUX_INITRD_ADDR_MAX_DEFAULT; ++ } ++ ++ AvailableMem = QEMU_LOW_MEMORY_SPLIT - QEMU_ACPI_DATA_SIZE; ++ if (InitrdMax >= AvailableMem) { ++ InitrdMax = AvailableMem - 1; ++ } ++ ++ if (InitrdSize >= InitrdMax) { ++ DEBUG (( ++ DEBUG_WARN, ++ "%a: an initrd of 0x%x bytes does not fit below 0x%x\n", ++ __func__, ++ InitrdSize, ++ InitrdMax ++ )); ++ return; + } + -+ Data[LINUX_HDR_LOADFLAGS_OFFSET] &= (UINT8)~LINUX_LOADFLAGS_CAN_USE_HEAP; ++ LinuxHdrWrite32 ( ++ Data, ++ LINUX_HDR_RAMDISK_IMAGE_OFFSET, ++ (InitrdMax - InitrdSize) & ~(UINT32)(QEMU_INITRD_ALIGNMENT - 1) ++ ); ++ LinuxHdrWrite32 (Data, LINUX_HDR_RAMDISK_SIZE_OFFSET, InitrdSize); +} + STATIC KERNEL_BLOB_ITEMS mKernelBlobItems[] = { { L"kernel", -@@ -1063,6 +1168,10 @@ +@@ -1063,6 +1278,10 @@ QemuKernelFetchBlob ( ChunkData += BlobItems->FwCfgItem[Idx].Size; } diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb index 0e5d89ed2..c946f87d0 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb @@ -89,10 +89,11 @@ DEPENDS = "nasm-native acpica-native ovmf-native util-linux-native" # modules / no sevsecret). The patch fails loud if a future edk2 bump changes # the AmdSev layout. # -# 0007-OvmfPkg-QemuKernelLoaderFsDxe-normalize-setup-header.patch zeroes the -# setup-header fields QEMU writes as boot loader, so RTMR[1] no longer depends -# on the host's QEMU version. It pairs with os/image/normalize-kernel-header.py, -# which applies the same normalization to the shipped bzImage. +# 0007-OvmfPkg-QemuKernelLoaderFsDxe-normalize-setup-header.patch writes the +# setup-header fields QEMU writes as boot loader, with the values QEMU <= 10.1 +# writes, so RTMR[1] no longer depends on the host's QEMU version. It pairs with +# os/image/normalize-kernel-header.py, which writes the same values into the +# shipped bzImage. OVMF_BUILD_SEV ??= "1" EDK_TOOLS_DIR="edk2_basetools" diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/dstack-tee-simulator.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/dstack-tee-simulator.bb index de42fa5ee..b222818fd 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/dstack-tee-simulator.bb +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/dstack-tee-simulator.bb @@ -26,9 +26,15 @@ EXTRA_CARGO_FLAGS = "-p dstack-tee-simulator" inherit cargo_bin +# The staged source lives outside WORKDIR and carries the host checkout's +# ownership. WORKDIR staging has no use for it, and applying it fails outright +# under container / user-namespace builds where the source uid/gid is not +# settable (rsync aborts with "chown/chgrp ... Invalid argument", exit 23, and +# leaves the partially-attributed temp file behind). Drop owner and group, keep +# the rest of archive mode -- permissions still matter for executable bits. do_unpack() { install -d "${S}" - rsync -a --exclude=".git" --exclude=".worktrees" --exclude="target" \ + rsync -a --no-owner --no-group --exclude=".git" --exclude=".worktrees" --exclude="target" \ "${DSTACK_CORE_SRC}/" "${S}/" } diff --git a/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0007-sched-smp-batch-wake-q-ipis-in-tdx-guests.patch b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0007-sched-smp-batch-wake-q-ipis-in-tdx-guests.patch new file mode 100644 index 000000000..27d48c6cd --- /dev/null +++ b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0007-sched-smp-batch-wake-q-ipis-in-tdx-guests.patch @@ -0,0 +1,387 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Kevin Wang +Date: Mon, 14 Sep 2026 10:11:06 -0700 +Subject: [PATCH 7/8] sched/smp: batch wake_up_q() notification IPIs in TDX + guests + +Every IPI sent by a TDX guest costs a TD exit and a TD entry. When a +task wakes many sleeping tasks through wake_up_q() (futex wake, +condition variable broadcast, OpenMP thread pools), each wakee on a +halted CPU costs one separate IPI hypercall, and the waker spends most +of its time in TD transitions instead of issuing wakeups. + +Add a runtime switch, tdx_wake_q_batch +(/sys/module/kernel/parameters/tdx_wake_q_batch), that keeps publishing +each wakeup immediately but defers the notification: target CPUs are +collected in a stack-local mask and notified with one mask IPI. A group +ends after 8 wakeups, after about 50 microseconds of sender time or when +the sender needs to reschedule, so the first wakee of a group waits at most +for the rest of that small group to be queued. Wakees that are polling in +idle still need no IPI at all. + +Batched TTWU entries use a separate per-CPU llist so that a synchronous +call-single callback is never hidden behind a deferred notification. The +IPI handler and the explicit idle, migration and CPU-down flush paths drain +both lists. Only task-context wake_up_q() and fair-class wakeups take the +batched path; RT, deadline, stop and ext tasks, interrupt-context wakeups +and direct try_to_wake_up() callers keep immediate notification. The switch +can only be enabled in a TDX guest without PREEMPT_RT. It is on by default +in TDX guests from a late initcall, so early boot and SMP bring-up keep +immediate notification; an explicit tdx_wake_q_batch= boot parameter is +preserved. + +With guest halt polling enabled in a 32-vCPU TDX guest, enabling batching +raised a PyTorch CPU operator with 32 passive OpenMP threads from 257 to +4440 MiB/s and doubled two many-thread pool synchronization benchmarks. + +Upstream-Status: Inappropriate [dstack TDX guest tuning, not submitted] +--- + kernel/sched/core.c | 131 +++++++++++++++++++++++++++++++++--- + kernel/sched/wake_q_batch.h | 16 +++++ + kernel/smp.c | 49 +++++++++++++- + 3 files changed, 184 insertions(+), 12 deletions(-) + create mode 100644 kernel/sched/wake_q_batch.h + +diff --git a/kernel/sched/core.c b/kernel/sched/core.c +index b5b5897b5..ece2b671b 100644 +--- a/kernel/sched/core.c ++++ b/kernel/sched/core.c +@@ -62,6 +62,7 @@ + #include + #include + #include ++#include "wake_q_batch.h" + #include + #include + #include +@@ -1070,10 +1071,109 @@ void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task) + put_task_struct(task); + } + ++/* ++ * TDX guests pay a full TD exit for every wakeup IPI. When enabled, wake_up_q() ++ * publishes each queued wakeup immediately but coalesces the notification IPIs ++ * of a small group into one mask IPI. A group ends after ++ * WAKE_Q_BATCH_TASKS wakeups, WAKE_Q_BATCH_NS of sender time or a pending ++ * reschedule, so the extra delay for the first wakee stays small. ++ */ ++#define WAKE_Q_BATCH_TASKS 8 ++#define WAKE_Q_BATCH_NS 50000 ++static bool tdx_wake_q_batch; ++/* Set once the boot command line or sysfs chose a value explicitly. */ ++static bool tdx_wake_q_batch_set; ++ ++static bool wake_q_batch_available(void) ++{ ++#if defined(CONFIG_X86_64) && defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_RT) ++ return cpu_feature_enabled(X86_FEATURE_TDX_GUEST); ++#else ++ return false; ++#endif ++} ++ ++static int param_set_tdx_wake_q_batch(const char *val, ++ const struct kernel_param *kp) ++{ ++ bool enabled; ++ int ret; ++ ++ ret = kstrtobool(val ? val : "1", &enabled); ++ if (ret) ++ return ret; ++ if (enabled && !wake_q_batch_available()) ++ return -EOPNOTSUPP; ++ WRITE_ONCE(*(bool *)kp->arg, enabled); ++ tdx_wake_q_batch_set = true; ++ return 0; ++} ++ ++static const struct kernel_param_ops tdx_wake_q_batch_ops = { ++ .flags = KERNEL_PARAM_OPS_FL_NOARG, ++ .set = param_set_tdx_wake_q_batch, ++ .get = param_get_bool, ++}; ++__core_param_cb(tdx_wake_q_batch, &tdx_wake_q_batch_ops, &tdx_wake_q_batch, 0644); ++ ++/* ++ * Enabled by default in TDX guests, but only once boot is complete, so early ++ * boot and SMP bring-up keep immediate notification. ++ */ ++static int __init tdx_wake_q_batch_default(void) ++{ ++ if (!tdx_wake_q_batch_set && wake_q_batch_available()) ++ WRITE_ONCE(tdx_wake_q_batch, true); ++ return 0; ++} ++late_initcall(tdx_wake_q_batch_default); ++ ++static int try_to_wake_up_mask(struct task_struct *p, unsigned int state, ++ int wake_flags, struct cpumask *mask); ++ ++static void wake_up_q_batch(struct wake_q_head *head) ++{ ++ struct wake_q_node *node = head->first; ++ struct task_struct *refs[WAKE_Q_BATCH_TASKS]; ++ cpumask_t mask; ++ unsigned int nr, i; ++ u64 start; ++ ++ while (node != WAKE_Q_TAIL) { ++ /* Keep CPU-down exclusion until all published wakeups are kicked. */ ++ preempt_disable(); ++ cpumask_clear(&mask); ++ nr = 0; ++ start = local_clock(); ++ do { ++ struct task_struct *task; ++ ++ task = container_of(node, struct task_struct, wake_q); ++ node = node->next; ++ /* Same publication and state barriers as wake_up_q(). */ ++ WRITE_ONCE(task->wake_q.next, NULL); ++ try_to_wake_up_mask(task, TASK_NORMAL, 0, &mask); ++ refs[nr++] = task; ++ } while (node != WAKE_Q_TAIL && nr < WAKE_Q_BATCH_TASKS && ++ !need_resched() && local_clock() - start < WAKE_Q_BATCH_NS); ++ ++ /* No batching state survives preempt_enable() or task release. */ ++ sched_ttwu_flush_batch(&mask); ++ preempt_enable(); ++ for (i = 0; i < nr; i++) ++ put_task_struct(refs[i]); ++ } ++} ++ + void wake_up_q(struct wake_q_head *head) + { + struct wake_q_node *node = head->first; + ++ if (READ_ONCE(tdx_wake_q_batch) && in_task()) { ++ wake_up_q_batch(head); ++ return; ++ } ++ + while (node != WAKE_Q_TAIL) { + struct task_struct *task; + +@@ -3846,7 +3946,8 @@ bool call_function_single_prep_ipi(int cpu) + * via sched_ttwu_wakeup() for activation so the wakee incurs the cost + * of the wakeup instead of the waker. + */ +-static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) ++static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags, ++ struct cpumask *mask) + { + struct rq *rq = cpu_rq(cpu); + +@@ -3854,7 +3955,11 @@ static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags + + WRITE_ONCE(rq->ttwu_pending, 1); + #ifdef CONFIG_SMP +- __smp_call_single_queue(cpu, &p->wake_entry.llist); ++ /* Publish under pi_lock: migration_cpu_stop() must already see this. */ ++ if (mask && p->sched_class == &fair_sched_class) ++ sched_ttwu_queue_batch(cpu, &p->wake_entry.llist, mask); ++ else ++ __smp_call_single_queue(cpu, &p->wake_entry.llist); + #endif + } + +@@ -3950,23 +4055,25 @@ static inline bool ttwu_queue_cond(struct task_struct *p, int cpu) + return false; + } + +-static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) ++static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags, ++ struct cpumask *mask) + { + if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(p, cpu)) { + sched_clock_cpu(cpu); /* Sync clocks across CPUs */ +- __ttwu_queue_wakelist(p, cpu, wake_flags); ++ __ttwu_queue_wakelist(p, cpu, wake_flags, mask); + return true; + } + + return false; + } + +-static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) ++static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags, ++ struct cpumask *mask) + { + struct rq *rq = cpu_rq(cpu); + struct rq_flags rf; + +- if (ttwu_queue_wakelist(p, cpu, wake_flags)) ++ if (ttwu_queue_wakelist(p, cpu, wake_flags, mask)) + return; + + rq_lock(rq, &rf); +@@ -4145,7 +4252,8 @@ bool ttwu_state_match(struct task_struct *p, unsigned int state, int *success) + * Return: %true if @p->state changes (an actual wakeup was done), + * %false otherwise. + */ +-int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) ++static int try_to_wake_up_mask(struct task_struct *p, unsigned int state, ++ int wake_flags, struct cpumask *mask) + { + guard(preempt)(); + int cpu, success = 0; +@@ -4269,7 +4377,7 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) + * scheduling. + */ + if (smp_load_acquire(&p->on_cpu) && +- ttwu_queue_wakelist(p, task_cpu(p), wake_flags)) ++ ttwu_queue_wakelist(p, task_cpu(p), wake_flags, mask)) + break; + + /* +@@ -4295,7 +4403,7 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) + set_task_cpu(p, cpu); + } + +- ttwu_queue(p, cpu, wake_flags); ++ ttwu_queue(p, cpu, wake_flags, mask); + } + out: + if (success) +@@ -4304,6 +4412,11 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) + return success; + } + ++int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) ++{ ++ return try_to_wake_up_mask(p, state, wake_flags, NULL); ++} ++ + static bool __task_needs_rq_lock(struct task_struct *p) + { + unsigned int state = READ_ONCE(p->__state); +diff --git a/kernel/sched/wake_q_batch.h b/kernel/sched/wake_q_batch.h +new file mode 100644 +index 000000000..73212d061 +--- /dev/null ++++ b/kernel/sched/wake_q_batch.h +@@ -0,0 +1,16 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef _KERNEL_SCHED_WAKE_Q_BATCH_H ++#define _KERNEL_SCHED_WAKE_Q_BATCH_H ++ ++struct cpumask; ++struct llist_node; ++ ++#ifdef CONFIG_SMP ++void sched_ttwu_queue_batch(int cpu, struct llist_node *node, ++ struct cpumask *mask); ++void sched_ttwu_flush_batch(struct cpumask *mask); ++#else ++static inline void sched_ttwu_flush_batch(struct cpumask *mask) { } ++#endif ++ ++#endif /* _KERNEL_SCHED_WAKE_Q_BATCH_H */ +diff --git a/kernel/smp.c b/kernel/smp.c +index 02f52291f..50593533e 100644 +--- a/kernel/smp.c ++++ b/kernel/smp.c +@@ -34,6 +34,7 @@ + + #include "smpboot.h" + #include "sched/smp.h" ++#include "sched/wake_q_batch.h" + + #define CSD_TYPE(_csd) ((_csd)->node.u_flags & CSD_FLAG_TYPE_MASK) + +@@ -46,6 +47,8 @@ struct call_function_data { + static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data); + + static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue); ++/* Batched TTWU wakeups; kept apart so no synchronous CSD waits on a deferred kick. */ ++static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, wake_q_batch_queue); + + static DEFINE_PER_CPU(atomic_t, trigger_backtrace) = ATOMIC_INIT(1); + +@@ -105,8 +108,10 @@ void __init call_function_init(void) + { + int i; + +- for_each_possible_cpu(i) ++ for_each_possible_cpu(i) { + init_llist_head(&per_cpu(call_single_queue, i)); ++ init_llist_head(&per_cpu(wake_q_batch_queue, i)); ++ } + + smpcfd_prepare_cpu(smp_processor_id()); + } +@@ -413,6 +418,33 @@ void __smp_call_single_queue(int cpu, struct llist_node *node) + send_call_function_single_ipi(cpu); + } + ++/* ++ * Queue a TTWU entry for a batched wakeup: the entry is published right away, ++ * only the notification is deferred by adding @cpu to @mask. ++ */ ++void sched_ttwu_queue_batch(int cpu, struct llist_node *node, ++ struct cpumask *mask) ++{ ++ lockdep_assert_preemption_disabled(); ++ if (trace_csd_queue_cpu_enabled()) { ++ call_single_data_t *csd; ++ ++ csd = container_of(node, call_single_data_t, node.llist); ++ trace_csd_queue_cpu(cpu, _RET_IP_, sched_ttwu_pending, csd); ++ } ++ if (llist_add(node, &per_cpu(wake_q_batch_queue, cpu)) && ++ call_function_single_prep_ipi(cpu)) ++ cpumask_set_cpu(cpu, mask); ++} ++ ++void sched_ttwu_flush_batch(struct cpumask *mask) ++{ ++ lockdep_assert_preemption_disabled(); ++ if (!cpumask_empty(mask)) ++ send_call_function_ipi_mask(mask); ++ cpumask_clear(mask); ++} ++ + /* + * Insert a previously allocated call_single_data_t element + * for execution on the given CPU. data must already have +@@ -552,7 +584,7 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline) + } + + if (!entry) +- return; ++ goto batch_wakes; + + /* + * Second; run all !SYNC callbacks. +@@ -592,6 +624,16 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline) + csd = llist_entry(entry, typeof(*csd), node.llist); + csd_do_func(sched_ttwu_pending, entry, csd); + } ++ ++batch_wakes: ++ /* Batched wakeups; also reached from idle, migration and CPU-down flushes. */ ++ entry = llist_del_all(this_cpu_ptr(&wake_q_batch_queue)); ++ entry = llist_reverse_order(entry); ++ if (entry) { ++ WARN_ON_ONCE(warn_cpu_offline && !cpu_online(smp_processor_id())); ++ csd = llist_entry(entry, typeof(*csd), node.llist); ++ csd_do_func(sched_ttwu_pending, entry, csd); ++ } + } + + +@@ -612,7 +654,8 @@ void flush_smp_call_function_queue(void) + unsigned int was_pending; + unsigned long flags; + +- if (llist_empty(this_cpu_ptr(&call_single_queue))) ++ if (llist_empty(this_cpu_ptr(&call_single_queue)) && ++ llist_empty(this_cpu_ptr(&wake_q_batch_queue))) + return; + + local_irq_save(flags); +-- +2.43.0 + diff --git a/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0008-x86-kvm-runtime-pv-single-target-ipi-in-tdx-guests.patch b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0008-x86-kvm-runtime-pv-single-target-ipi-in-tdx-guests.patch new file mode 100644 index 000000000..d02fc3763 --- /dev/null +++ b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0008-x86-kvm-runtime-pv-single-target-ipi-in-tdx-guests.patch @@ -0,0 +1,110 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Kevin Wang +Date: Mon, 14 Sep 2026 10:11:07 -0700 +Subject: [PATCH 8/8] x86/kvm: allow PV single-target function IPIs in TDX + guests at runtime + +In a TDX guest a single-target IPI written to the x2APIC ICR is emulated +through a #VE and a TDVMCALL. The KVM PV IPI hypercall avoids the #VE step, +but only the mask IPI callbacks use it; CALL_FUNCTION_SINGLE IPIs, which +carry cross-CPU wakeups, still go through the emulated ICR write. + +Add tdx_pv_single_ipi (boot parameter and +/sys/module/kernel/parameters/tdx_pv_single_ipi) that routes +smp_ops.send_call_func_single_ipi through the PV mask IPI path. Both +callbacks are built in and valid at any time, so the pointer is simply +switched. Reschedule, NMI and shorthand IPIs are unchanged. The option is +only accepted in a TDX guest and takes effect once PV IPIs are set up. It +is on by default in TDX guests unless the boot command line sets it. + +The benefit depends on how many wakeup IPIs the workload still sends: in +default 32-vCPU and 248-vCPU TDX guests it improved synchronization-heavy +benchmarks by about 32% and 10% (geometric mean), but with guest halt +polling and wake-queue batching it was neutral to slightly negative, so it +stays a runtime switch that guests which enable halt polling can turn off. + +Upstream-Status: Inappropriate [dstack TDX guest tuning, not submitted] +--- + arch/x86/kernel/kvm.c | 59 +++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 59 insertions(+) + +diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c +index b67d7c59d..e3372ee3e 100644 +--- a/arch/x86/kernel/kvm.c ++++ b/arch/x86/kernel/kvm.c +@@ -561,6 +561,57 @@ static void kvm_send_ipi_mask(const struct cpumask *mask, int vector) + __send_ipi_mask(mask, vector); + } + ++/* ++ * A single-target x2APIC ICR write in a TDX guest is emulated through #VE and ++ * a TDVMCALL. Optionally send CALL_FUNCTION_SINGLE IPIs, which carry ++ * cross-CPU wakeups, through the PV IPI hypercall instead. Reschedule, NMI ++ * and shorthand IPIs are unchanged. On by default in TDX guests; it can be ++ * toggled at runtime through /sys/module/kernel/parameters/tdx_pv_single_ipi. ++ */ ++static bool tdx_pv_single_ipi; ++/* Set once the boot command line or sysfs chose a value explicitly. */ ++static bool tdx_pv_single_ipi_set; ++static bool kvm_pv_ipi_ready; ++static void (*kvm_native_single_ipi)(int cpu); ++ ++static void kvm_send_call_func_single_ipi(int cpu) ++{ ++ __send_ipi_mask(cpumask_of(cpu), CALL_FUNCTION_SINGLE_VECTOR); ++} ++ ++static void kvm_apply_pv_single_ipi(void) ++{ ++ /* Both callbacks are built in and valid at all times. */ ++ WRITE_ONCE(smp_ops.send_call_func_single_ipi, ++ READ_ONCE(tdx_pv_single_ipi) ? kvm_send_call_func_single_ipi : ++ kvm_native_single_ipi); ++} ++ ++static int param_set_tdx_pv_single_ipi(const char *val, ++ const struct kernel_param *kp) ++{ ++ bool enabled; ++ int ret; ++ ++ ret = kstrtobool(val ? val : "1", &enabled); ++ if (ret) ++ return ret; ++ if (enabled && !cpu_feature_enabled(X86_FEATURE_TDX_GUEST)) ++ return -EOPNOTSUPP; ++ WRITE_ONCE(tdx_pv_single_ipi, enabled); ++ tdx_pv_single_ipi_set = true; ++ if (READ_ONCE(kvm_pv_ipi_ready)) ++ kvm_apply_pv_single_ipi(); ++ return 0; ++} ++ ++static const struct kernel_param_ops tdx_pv_single_ipi_ops = { ++ .flags = KERNEL_PARAM_OPS_FL_NOARG, ++ .set = param_set_tdx_pv_single_ipi, ++ .get = param_get_bool, ++}; ++__core_param_cb(tdx_pv_single_ipi, &tdx_pv_single_ipi_ops, &tdx_pv_single_ipi, 0644); ++ + static void kvm_send_ipi_mask_allbutself(const struct cpumask *mask, int vector) + { + unsigned int this_cpu = smp_processor_id(); +@@ -629,6 +680,14 @@ static __init void kvm_setup_pv_ipi(void) + { + apic_update_callback(send_IPI_mask, kvm_send_ipi_mask); + apic_update_callback(send_IPI_mask_allbutself, kvm_send_ipi_mask_allbutself); ++ if (cpu_feature_enabled(X86_FEATURE_TDX_GUEST)) { ++ kvm_native_single_ipi = smp_ops.send_call_func_single_ipi; ++ /* On by default in TDX guests unless the command line chose. */ ++ if (!tdx_pv_single_ipi_set) ++ WRITE_ONCE(tdx_pv_single_ipi, true); ++ WRITE_ONCE(kvm_pv_ipi_ready, true); ++ kvm_apply_pv_single_ipi(); ++ } + pr_info("setup PV IPIs\n"); + } + +-- +2.43.0 + diff --git a/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0009-mm-page-alloc-keep-accepted-reserve-before-watermarks.patch b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0009-mm-page-alloc-keep-accepted-reserve-before-watermarks.patch new file mode 100644 index 000000000..ae02c62f3 --- /dev/null +++ b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0009-mm-page-alloc-keep-accepted-reserve-before-watermarks.patch @@ -0,0 +1,135 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Kevin Wang +Date: Mon, 14 Sep 2026 20:30:52 -0700 +Subject: [PATCH 09/10] mm/page_alloc: keep a small accepted reserve before + watermarks are initialized + +Until init_per_zone_wmark_min() runs as a postcore initcall, the zone +watermarks are zero and cond_accept_memory() accepts one MAX_ORDER page +whenever it is called. get_page_from_freelist() and alloc_pages_bulk() +call it before every allocation, so each page allocation made during +early boot, including all of smp_init(), accepts another 4 MiB even +when the zone has plenty of accepted free memory. + +This is expensive for TDX guests on hosts that map private memory at +4K only. Accepting 4 MiB then takes 1024 TDG.MEM.PAGE.ACCEPT calls, each +one a TD exit and a TDH.MEM.PAGE.AUG. The number of allocations grows +with the number of CPUs brought up, and each exit gets slower as more +idle vCPUs take the periodic tick. On a Granite Rapids host running a +7.0 kernel, a 248-vCPU, 32 GiB TD accepts about 13 GiB during SMP +bring-up alone and needs 575-680 seconds to reach init. + +Before the watermarks are initialized, skip the proactive accept while +the zone still has at least two MAX_ORDER pages of accepted free memory. +Callers that retry after a failed watermark check or a failed allocation +pass force=true and still accept one MAX_ORDER page, so an allocation +that cannot be served from accepted memory makes progress as before. +Nothing changes once the watermarks are initialized. + +With this change the same TD brings up all 248 CPUs in about 2 seconds +and reaches init in 114 seconds. +--- + mm/page_alloc.c | 30 +++++++++++++++++++----------- + 1 file changed, 19 insertions(+), 11 deletions(-) + +diff --git a/mm/page_alloc.c b/mm/page_alloc.c +index 775e02b79..08d411add 100644 +--- a/mm/page_alloc.c ++++ b/mm/page_alloc.c +@@ -318,7 +318,7 @@ EXPORT_SYMBOL(nr_online_nodes); + + static bool page_contains_unaccepted(struct page *page, unsigned int order); + static bool cond_accept_memory(struct zone *zone, unsigned int order, +- int alloc_flags); ++ int alloc_flags, bool force); + static bool __free_unaccepted(struct page *page); + + int page_group_by_mobility_disabled __read_mostly; +@@ -3854,7 +3854,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, + continue; + } + +- cond_accept_memory(zone, order, alloc_flags); ++ cond_accept_memory(zone, order, alloc_flags, false); + + /* + * Detect whether the number of free pages is below high +@@ -3881,7 +3881,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, + gfp_mask)) { + int ret; + +- if (cond_accept_memory(zone, order, alloc_flags)) ++ if (cond_accept_memory(zone, order, alloc_flags, true)) + goto try_this_zone; + + /* +@@ -3934,7 +3934,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, + + return page; + } else { +- if (cond_accept_memory(zone, order, alloc_flags)) ++ if (cond_accept_memory(zone, order, alloc_flags, true)) + goto try_this_zone; + + /* Try again if zone has deferred pages */ +@@ -5118,7 +5118,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid, + goto failed; + } + +- cond_accept_memory(zone, 0, alloc_flags); ++ cond_accept_memory(zone, 0, alloc_flags, false); + retry_this_zone: + mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK) + nr_pages; + if (zone_watermark_fast(zone, 0, mark, +@@ -5127,7 +5127,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid, + break; + } + +- if (cond_accept_memory(zone, 0, alloc_flags)) ++ if (cond_accept_memory(zone, 0, alloc_flags, true)) + goto retry_this_zone; + + /* Try again if zone has deferred pages */ +@@ -7522,7 +7522,7 @@ static bool try_to_accept_memory_one(struct zone *zone) + } + + static bool cond_accept_memory(struct zone *zone, unsigned int order, +- int alloc_flags) ++ int alloc_flags, bool force) + { + long to_accept, wmark; + bool ret = false; +@@ -7539,10 +7539,18 @@ static bool cond_accept_memory(struct zone *zone, unsigned int order, + /* + * Watermarks have not been initialized yet. + * +- * Accepting one MAX_ORDER page to ensure progress. +- */ +- if (!wmark) ++ * Keep a small reserve of accepted free memory instead of accepting ++ * one MAX_ORDER page on every allocation. Accept one MAX_ORDER page ++ * when the allocation could not be satisfied, to ensure progress. ++ */ ++ if (!wmark) { ++ if (!force && ++ zone_page_state(zone, NR_FREE_PAGES) - ++ zone_page_state(zone, NR_UNACCEPTED) >= ++ 2 * MAX_ORDER_NR_PAGES) ++ return false; + return try_to_accept_memory_one(zone); ++ } + + /* How much to accept to get to promo watermark? */ + to_accept = wmark - +@@ -7586,7 +7594,7 @@ static bool page_contains_unaccepted(struct page *page, unsigned int order) + } + + static bool cond_accept_memory(struct zone *zone, unsigned int order, +- int alloc_flags) ++ int alloc_flags, bool force) + { + return false; + } +-- +2.43.0 + diff --git a/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0010-x86-tdx-convert-swiotlb-to-shared-after-smp-bringup.patch b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0010-x86-tdx-convert-swiotlb-to-shared-after-smp-bringup.patch new file mode 100644 index 000000000..896275a2d --- /dev/null +++ b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0010-x86-tdx-convert-swiotlb-to-shared-after-smp-bringup.patch @@ -0,0 +1,137 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Kevin Wang +Date: Mon, 14 Sep 2026 20:30:52 -0700 +Subject: [PATCH 10/10] x86/tdx: convert the SWIOTLB buffer to shared after SMP + bring-up + +mem_encrypt_init() converts the default SWIOTLB buffer from private to +shared before SMP bring-up. The buffer lives in memory the firmware has +already accepted, so the VMM has to remove every private page backing +it. For each removed page the VMM forces all vCPUs out of the TD, and at +this point the not-yet-woken APs busy-wait inside the TD in firmware, +so every removal waits for a TD exit from every AP. With 248 vCPUs the +default 1 GiB buffer takes about 68 seconds to convert. + +Defer the conversion for TDX guests to fs_initcall_sync. By then the +APs are online, idle in HLT outside the TD, and have stopped the +periodic tick because clocksource_done_booting() enabled high +resolution timers. Converting right after smp_init() is not enough: the +idle APs still take the jiffies tick and re-enter the TD about 1000 +times per second each, and the conversion took over 200 seconds. No +device maps the bounce buffer before device probing starts at +device_initcall. Other memory encryption platforms are unchanged. + +A bounce through the still-private default pool would fail silently: +the device reads a buffer it cannot decrypt. Record in SWIOTLB when +swiotlb_update_mem_attributes() has run and WARN_ON_ONCE() in +swiotlb_tbl_map_single() if the default pool is used before that on a +platform with memory encryption, so a future early DMA user is caught +instead of corrupting data. + +With this change the conversion takes about 6 seconds and the 248-vCPU +TD reaches init in 39-51 seconds. +--- + arch/x86/mm/mem_encrypt.c | 30 ++++++++++++++++++++++++++++-- + kernel/dma/swiotlb.c | 25 +++++++++++++++++++++---- + 2 files changed, 49 insertions(+), 6 deletions(-) + +diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c +index 95bae74fd..48618d2dd 100644 +--- a/arch/x86/mm/mem_encrypt.c ++++ b/arch/x86/mm/mem_encrypt.c +@@ -85,14 +85,40 @@ static void print_mem_encrypt_feature_info(void) + } + } + ++/* ++ * Converting the SWIOTLB buffer to shared makes the VMM remove every private ++ * page it backs. For each page the VMM has to force all vCPUs out of the TD, ++ * and before SMP bring-up the not-yet-woken APs are busy-waiting inside the ++ * TD in firmware, so every removal waits for an exit from every AP. With ++ * hundreds of vCPUs this takes minutes for the default 1 GiB buffer. Once ++ * the APs are online they idle in HLT outside the TD and the removal is ++ * cheap only after they have also stopped the periodic tick, which happens ++ * once clocksource_done_booting() (fs_initcall) enables high resolution ++ * timers and NOHZ. No device uses the bounce buffer before device probing ++ * starts, so do the conversion from fs_initcall_sync. swiotlb_tbl_map_single() ++ * warns if a bounce through the default pool ever happens before that. ++ */ ++static int __init tdx_swiotlb_update_mem_attributes(void) ++{ ++ if (cpu_feature_enabled(X86_FEATURE_TDX_GUEST)) ++ swiotlb_update_mem_attributes(); ++ return 0; ++} ++fs_initcall_sync(tdx_swiotlb_update_mem_attributes); ++ + /* Architecture __weak replacement functions */ + void __init mem_encrypt_init(void) + { + if (!cc_platform_has(CC_ATTR_MEM_ENCRYPT)) + return; + +- /* Call into SWIOTLB to update the SWIOTLB DMA buffers */ +- swiotlb_update_mem_attributes(); ++ /* ++ * Call into SWIOTLB to update the SWIOTLB DMA buffers. TDX guests ++ * defer this until secondary CPUs are online, see ++ * tdx_swiotlb_update_mem_attributes(). ++ */ ++ if (!cpu_feature_enabled(X86_FEATURE_TDX_GUEST)) ++ swiotlb_update_mem_attributes(); + + snp_secure_tsc_prepare(); + +diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c +index fa7377b01..bc0900f1d 100644 +--- a/kernel/dma/swiotlb.c ++++ b/kernel/dma/swiotlb.c +@@ -250,6 +250,13 @@ static inline unsigned long nr_slots(u64 val) + return DIV_ROUND_UP(val, IO_TLB_SIZE); + } + ++/* ++ * Set once swiotlb_update_mem_attributes() has run. On platforms with memory ++ * encryption the default pool is private until then, so a bounce through it ++ * would hand the device a buffer it cannot read and fail silently. ++ */ ++static bool swiotlb_default_pool_decrypted __ro_after_init; ++ + /* + * Early SWIOTLB allocation may be too early to allow an architecture to + * perform the desired operations. This function allows the architecture to +@@ -261,10 +268,12 @@ void __init swiotlb_update_mem_attributes(void) + struct io_tlb_pool *mem = &io_tlb_default_mem.defpool; + unsigned long bytes; + +- if (!mem->nslabs || mem->late_alloc) +- return; +- bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT); +- set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT); ++ if (mem->nslabs && !mem->late_alloc) { ++ bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT); ++ set_memory_decrypted((unsigned long)mem->vaddr, ++ bytes >> PAGE_SHIFT); ++ } ++ swiotlb_default_pool_decrypted = true; + } + + static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start, +@@ -1435,6 +1444,14 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr, + return (phys_addr_t)DMA_MAPPING_ERROR; + } + ++ /* ++ * Catch a bounce through the default pool before it has been made ++ * shared with the device, see swiotlb_update_mem_attributes(). ++ */ ++ WARN_ON_ONCE(pool == &io_tlb_default_mem.defpool && ++ !swiotlb_default_pool_decrypted && ++ cc_platform_has(CC_ATTR_MEM_ENCRYPT)); ++ + /* + * If dma_skip_sync was set, reset it on first SWIOTLB buffer + * mapping to always sync SWIOTLB buffers. +-- +2.43.0 + diff --git a/sdk/js/src/__tests__/solana.test.ts b/sdk/js/src/__tests__/solana.test.ts index 87436b3bb..f9eb612b5 100644 --- a/sdk/js/src/__tests__/solana.test.ts +++ b/sdk/js/src/__tests__/solana.test.ts @@ -2,83 +2,37 @@ // // SPDX-License-Identifier: Apache-2.0 - -import { expect, describe, it, vi } from 'vitest' +import { expect, describe, it } from 'vitest' import { Keypair } from '@solana/web3.js' - -import { DstackClientV0, TappdClient } from '../index' +import type { GetKeyResponse, GetTlsKeyResponse } from '../client-v0' import { toKeypair, toKeypairSecure } from '../solana' describe('solana support', () => { - describe('toKeypair (legacy)', () => { - it('should able to get keypair from getKey with DstackClientV0', async () => { - const client = new DstackClientV0() - const result = await client.getKey('/', 'test') - const keypair = toKeypair(result) - expect(keypair).toBeInstanceOf(Keypair) - expect(keypair.secretKey.length).toBe(64) - }) - - it('should able to get keypair from deriveKey with TappdClient', async () => { - const client = new TappdClient() - const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - - const result = await client.deriveKey('/', 'test') - const keypair = toKeypair(result) - expect(keypair).toBeInstanceOf(Keypair) - expect(keypair.secretKey.length).toBe(64) - expect(consoleSpy).toHaveBeenCalledWith('toKeypair: Please don\'t use `deriveKey` method to get key, use `getKey` instead.') - - consoleSpy.mockRestore() - }) - - it('should able to get keypair from getTlsKey with DstackClientV0', async () => { - const client = new DstackClientV0() - const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - - const result = await client.getTlsKey() - const keypair = toKeypair(result) - expect(keypair).toBeInstanceOf(Keypair) - expect(keypair.secretKey.length).toBe(64) - expect(consoleSpy).toHaveBeenCalledWith('toKeypair: Please don\'t use `deriveKey` method to get key, use `getKey` instead.') - - consoleSpy.mockRestore() - }) - }) - - describe('toKeypairSecure', () => { - it('should able to get keypair from getKey with DstackClientV0', async () => { - const client = new DstackClientV0() - const result = await client.getKey('/', 'test') - const keypair = toKeypairSecure(result) - expect(keypair).toBeInstanceOf(Keypair) - expect(keypair.secretKey.length).toBe(64) - }) - - it('should able to get keypair from deriveKey with TappdClient', async () => { - const client = new TappdClient() - const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - - const result = await client.deriveKey('/', 'test') - const keypair = toKeypairSecure(result) - expect(keypair).toBeInstanceOf(Keypair) - expect(keypair.secretKey.length).toBe(64) - expect(consoleSpy).toHaveBeenCalledWith('toKeypairSecure: Please don\'t use `deriveKey` method to get key, use `getKey` instead.') - - consoleSpy.mockRestore() - }) - - it('should able to get keypair from getTlsKey with DstackClientV0', async () => { - const client = new DstackClientV0() - const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - - const result = await client.getTlsKey() - const keypair = toKeypairSecure(result) - expect(keypair).toBeInstanceOf(Keypair) - expect(keypair.secretKey.length).toBe(64) - expect(consoleSpy).toHaveBeenCalledWith('toKeypairSecure: Please don\'t use `deriveKey` method to get key, use `getKey` instead.') - - consoleSpy.mockRestore() + for (const [name, adapter] of [ + ['toKeypair', toKeypair], + ['toKeypairSecure', toKeypairSecure], + ] as const) { + describe(name, () => { + it('creates a keypair from getKey', async () => { + const result: GetKeyResponse = { + __name__: 'GetKeyResponse', + key: new Uint8Array(32).fill(1), + signature_chain: [], + } + const keypair = adapter(result) + expect(keypair).toBeInstanceOf(Keypair) + expect(keypair.secretKey.length).toBe(64) + }) + + it('rejects TLS keys', async () => { + const result: GetTlsKeyResponse = { + __name__: 'GetTlsKeyResponse', + key: 'not used', + certificate_chain: [], + asUint8Array: () => new Uint8Array(), + } + expect(() => adapter(result as never)).toThrow(/TLS keys cannot be used/) + }) }) - }) + } }) diff --git a/sdk/js/src/__tests__/viem.test.ts b/sdk/js/src/__tests__/viem.test.ts index 1dbd705de..083a566e2 100644 --- a/sdk/js/src/__tests__/viem.test.ts +++ b/sdk/js/src/__tests__/viem.test.ts @@ -2,93 +2,36 @@ // // SPDX-License-Identifier: Apache-2.0 - -import { expect, describe, it, vi } from 'vitest' -import { DstackClientV0, TappdClient } from '../index' +import { expect, describe, it } from 'vitest' +import type { GetKeyResponse, GetTlsKeyResponse } from '../client-v0' import { toViemAccount, toViemAccountSecure } from '../viem' describe('viem support', () => { - describe('toViemAccount (legacy)', () => { - it('should able to get account from getKey with DstackClientV0', async () => { - const client = new DstackClientV0() - const result = await client.getKey('/', 'test') - const account = toViemAccount(result) - - expect(account.source).toBe('privateKey') - expect(typeof account.sign).toBe('function') - expect(typeof account.signMessage).toBe('function') - }) - - it('should able to get account from deriveKey with TappdClient', async () => { - const client = new TappdClient() - const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - - const result = await client.deriveKey('/', 'test') - const account = toViemAccount(result) - - expect(account.source).toBe('privateKey') - expect(typeof account.sign).toBe('function') - expect(typeof account.signMessage).toBe('function') - expect(consoleSpy).toHaveBeenCalledWith('toViemAccount: Please don\'t use `deriveKey` method to get key, use `getKey` instead.') - - consoleSpy.mockRestore() - }) - - it('should able to get account from getTlsKey with DstackClientV0', async () => { - const client = new DstackClientV0() - const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - - const result = await client.getTlsKey() - const account = toViemAccount(result) - - expect(account.source).toBe('privateKey') - expect(typeof account.sign).toBe('function') - expect(typeof account.signMessage).toBe('function') - expect(consoleSpy).toHaveBeenCalledWith('toViemAccount: Please don\'t use `deriveKey` method to get key, use `getKey` instead.') - - consoleSpy.mockRestore() - }) - }) - - describe('toViemAccountSecure', () => { - it('should able to get account from getKey with DstackClientV0', async () => { - const client = new DstackClientV0() - const result = await client.getKey('/', 'test') - const account = toViemAccountSecure(result) - - expect(account.source).toBe('privateKey') - expect(typeof account.sign).toBe('function') - expect(typeof account.signMessage).toBe('function') - }) - - it('should able to get account from deriveKey with TappdClient', async () => { - const client = new TappdClient() - const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - - const result = await client.deriveKey('/', 'test') - const account = toViemAccountSecure(result) - - expect(account.source).toBe('privateKey') - expect(typeof account.sign).toBe('function') - expect(typeof account.signMessage).toBe('function') - expect(consoleSpy).toHaveBeenCalledWith('toViemAccountSecure: Please don\'t use `deriveKey` method to get key, use `getKey` instead.') - - consoleSpy.mockRestore() - }) - - it('should able to get account from getTlsKey with DstackClientV0', async () => { - const client = new DstackClientV0() - const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - - const result = await client.getTlsKey() - const account = toViemAccountSecure(result) - - expect(account.source).toBe('privateKey') - expect(typeof account.sign).toBe('function') - expect(typeof account.signMessage).toBe('function') - expect(consoleSpy).toHaveBeenCalledWith('toViemAccountSecure: Please don\'t use `deriveKey` method to get key, use `getKey` instead.') - - consoleSpy.mockRestore() + for (const [name, adapter] of [ + ['toViemAccount', toViemAccount], + ['toViemAccountSecure', toViemAccountSecure], + ] as const) { + describe(name, () => { + it('creates an account from getKey', async () => { + const result: GetKeyResponse = { + __name__: 'GetKeyResponse', + key: new Uint8Array(32).fill(1), + signature_chain: [], + } + const account = adapter(result) + expect(account.source).toBe('privateKey') + expect(typeof account.sign).toBe('function') + }) + + it('rejects TLS keys', async () => { + const result: GetTlsKeyResponse = { + __name__: 'GetTlsKeyResponse', + key: 'not used', + certificate_chain: [], + asUint8Array: () => new Uint8Array(), + } + expect(() => adapter(result as never)).toThrow(/TLS keys cannot be used/) + }) }) - }) + } }) diff --git a/sdk/js/src/solana.ts b/sdk/js/src/solana.ts index 36e633451..057ccafa7 100644 --- a/sdk/js/src/solana.ts +++ b/sdk/js/src/solana.ts @@ -2,35 +2,25 @@ // // SPDX-License-Identifier: Apache-2.0 -import { sha256 } from '@noble/hashes/sha256' -import { type GetKeyResponse, type GetTlsKeyResponse } from './client-v0' +import { type GetKeyResponse } from './client-v0' import { Keypair } from '@solana/web3.js' -/** - * @deprecated use toKeypairSecure instead. This method has security concerns. - * Current implementation uses raw key material without proper hashing. - */ -export function toKeypair(keyResponse: GetTlsKeyResponse | GetKeyResponse) { - // Keep legacy behavior for GetTlsKeyResponse, but with warning. +function rejectTlsKey(keyResponse: { readonly __name__: string }): void { if (keyResponse.__name__ === 'GetTlsKeyResponse') { - console.warn('toKeypair: Please don\'t use `deriveKey` method to get key, use `getKey` instead.') - // Restored original behavior: using first 32 bytes directly - const bytes = keyResponse.asUint8Array(32) - return Keypair.fromSeed(bytes) + throw new TypeError('TLS keys cannot be used to derive Solana keypairs; use getKey()') } - return Keypair.fromSeed(keyResponse.key) } /** - * Creates a Solana Keypair from DeriveKeyResponse using secure key derivation. - * This method applies SHA256 hashing to the complete key material for enhanced security. + * @deprecated use toKeypairSecure instead. */ -export function toKeypairSecure(keyResponse: GetTlsKeyResponse | GetKeyResponse) { - // Keep legacy behavior for GetTlsKeyResponse, but with warning. - if (keyResponse.__name__ === 'GetTlsKeyResponse') { - console.warn('toKeypairSecure: Please don\'t use `deriveKey` method to get key, use `getKey` instead.') - const buf = sha256(keyResponse.asUint8Array()) - return Keypair.fromSeed(buf) - } +export function toKeypair(keyResponse: GetKeyResponse) { + rejectTlsKey(keyResponse) + return Keypair.fromSeed(keyResponse.key) +} + +/** Creates a Solana keypair from a getKey() response. */ +export function toKeypairSecure(keyResponse: GetKeyResponse) { + rejectTlsKey(keyResponse) return Keypair.fromSeed(keyResponse.key) } diff --git a/sdk/js/src/viem.ts b/sdk/js/src/viem.ts index 4f74bc524..e2bad3be8 100644 --- a/sdk/js/src/viem.ts +++ b/sdk/js/src/viem.ts @@ -2,37 +2,27 @@ // // SPDX-License-Identifier: Apache-2.0 -import { sha256 } from '@noble/hashes/sha256' -import { bytesToHex } from '@noble/hashes/utils' -import { type GetKeyResponse, type GetTlsKeyResponse } from './client-v0' +import { type GetKeyResponse } from './client-v0' import { privateKeyToAccount } from 'viem/accounts' -/** - * @deprecated use toViemAccountSecure instead. This method has security concerns. - * Current implementation uses raw key material without proper hashing. - */ -export function toViemAccount(keyResponse: GetKeyResponse | GetTlsKeyResponse) { - // Keep legacy behavior for GetTlsKeyResponse, but with warning. +function rejectTlsKey(keyResponse: { readonly __name__: string }): void { if (keyResponse.__name__ === 'GetTlsKeyResponse') { - console.warn('toViemAccount: Please don\'t use `deriveKey` method to get key, use `getKey` instead.') - const hex = Array.from(keyResponse.asUint8Array(32)).map(b => b.toString(16).padStart(2, '0')).join('') - return privateKeyToAccount(`0x${hex}`) + throw new TypeError('TLS keys cannot be used to derive Viem accounts; use getKey()') } - const hex = Array.from(keyResponse.key).map(b => b.toString(16).padStart(2, '0')).join('') - return privateKeyToAccount(`0x${hex}`) } /** - * Creates a Viem account from DeriveKeyResponse using secure key derivation. - * This method applies SHA256 hashing to the complete key material for enhanced security. + * @deprecated use toViemAccountSecure instead. */ -export function toViemAccountSecure(keyResponse: GetKeyResponse | GetTlsKeyResponse) { - // Keep legacy behavior for GetTlsKeyResponse, but with warning. - if (keyResponse.__name__ === 'GetTlsKeyResponse') { - console.warn('toViemAccountSecure: Please don\'t use `deriveKey` method to get key, use `getKey` instead.') - const hex = bytesToHex(sha256(keyResponse.asUint8Array())) - return privateKeyToAccount(`0x${hex}`) - } +export function toViemAccount(keyResponse: GetKeyResponse) { + rejectTlsKey(keyResponse) + const hex = Array.from(keyResponse.key).map(b => b.toString(16).padStart(2, '0')).join('') + return privateKeyToAccount(`0x${hex}`) +} + +/** Creates a Viem account from a getKey() response. */ +export function toViemAccountSecure(keyResponse: GetKeyResponse) { + rejectTlsKey(keyResponse) const hex = Array.from(keyResponse.key).map(b => b.toString(16).padStart(2, '0')).join('') return privateKeyToAccount(`0x${hex}`) } diff --git a/sdk/python/src/dstack_sdk/ethereum.py b/sdk/python/src/dstack_sdk/ethereum.py index 060bafb97..d673c329a 100644 --- a/sdk/python/src/dstack_sdk/ethereum.py +++ b/sdk/python/src/dstack_sdk/ethereum.py @@ -10,9 +10,6 @@ yourself. """ -import hashlib -import warnings - from eth_account import Account from eth_account.signers.local import LocalAccount @@ -20,49 +17,23 @@ from .dstack_client_v0 import GetTlsKeyResponse -def to_account(get_key_response: GetKeyResponse | GetTlsKeyResponse) -> LocalAccount: - """Create an Ethereum account from a DstackClientV0 key response. - - DEPRECATED: Use to_account_secure instead. This method has security concerns. - Current implementation uses raw key material without proper hashing. +def _reject_tls_key(response: GetKeyResponse | GetTlsKeyResponse) -> None: + if isinstance(response, GetTlsKeyResponse): + raise TypeError( + "TLS keys cannot be used to derive Ethereum accounts; use get_key()" + ) - Args: - get_key_response: Response from get_key() or get_tls_key() - Returns: - Account: Ethereum account object +def to_account(get_key_response: GetKeyResponse) -> LocalAccount: + """Create an Ethereum account from a DstackClientV0 get_key() response. + DEPRECATED: Use to_account_secure instead. """ - if isinstance(get_key_response, GetTlsKeyResponse): - warnings.warn( - "to_account: Please don't use getTlsKey method to get key, use getKey instead.", - DeprecationWarning, - stacklevel=2, - ) - key_bytes = get_key_response.as_uint8array(32) - return Account.from_key(key_bytes) # type: ignore[no-any-return] - else: # GetKeyResponse - return Account.from_key(get_key_response.decode_key()) # type: ignore[no-any-return] + _reject_tls_key(get_key_response) + return Account.from_key(get_key_response.decode_key()) # type: ignore[no-any-return] -def to_account_secure( - get_key_response: GetKeyResponse | GetTlsKeyResponse, -) -> LocalAccount: - """Create an Ethereum account using SHA256 of full key material for security.""" - if isinstance(get_key_response, GetTlsKeyResponse): - warnings.warn( - "to_account_secure: Please don't use getTlsKey method to get key, use getKey instead.", - DeprecationWarning, - stacklevel=2, - ) - try: - # Hash the complete key material with SHA256 - key_bytes = get_key_response.as_uint8array() - hashed_key = hashlib.sha256(key_bytes).digest() - return Account.from_key(hashed_key) # type: ignore[no-any-return] - except Exception as e: - raise RuntimeError( - "to_account_secure: missing SHA256 support, please upgrade your system" - ) from e - else: # GetKeyResponse - return Account.from_key(get_key_response.decode_key()) # type: ignore[no-any-return] +def to_account_secure(get_key_response: GetKeyResponse) -> LocalAccount: + """Create an Ethereum account from a DstackClientV0 get_key() response.""" + _reject_tls_key(get_key_response) + return Account.from_key(get_key_response.decode_key()) # type: ignore[no-any-return] diff --git a/sdk/python/src/dstack_sdk/solana.py b/sdk/python/src/dstack_sdk/solana.py index a33fdc953..c6acb6a8a 100644 --- a/sdk/python/src/dstack_sdk/solana.py +++ b/sdk/python/src/dstack_sdk/solana.py @@ -10,57 +10,29 @@ ``Keypair.from_seed`` yourself. """ -import hashlib -import warnings - from solders.keypair import Keypair from .dstack_client_v0 import GetKeyResponse from .dstack_client_v0 import GetTlsKeyResponse -def to_keypair(get_key_response: GetKeyResponse | GetTlsKeyResponse) -> Keypair: - """Create a Solana Keypair from a DstackClientV0 key response. - - DEPRECATED: Use to_keypair_secure instead. This method has security concerns. - Current implementation uses raw key material without proper hashing. +def _reject_tls_key(response: GetKeyResponse | GetTlsKeyResponse) -> None: + if isinstance(response, GetTlsKeyResponse): + raise TypeError( + "TLS keys cannot be used to derive Solana keypairs; use get_key()" + ) - Args: - get_key_response: Response from get_key() or get_tls_key() - Returns: - Keypair: Solana keypair object +def to_keypair(get_key_response: GetKeyResponse) -> Keypair: + """Create a Solana keypair from a DstackClientV0 get_key() response. + DEPRECATED: Use to_keypair_secure instead. """ - if isinstance(get_key_response, GetTlsKeyResponse): - warnings.warn( - "to_keypair: Please don't use getTlsKey method to get key, use getKey instead.", - DeprecationWarning, - stacklevel=2, - ) - # Restored original behavior: using first 32 bytes directly - key_bytes = get_key_response.as_uint8array(32) - return Keypair.from_seed(key_bytes) - else: # GetKeyResponse - return Keypair.from_seed(get_key_response.decode_key()) + _reject_tls_key(get_key_response) + return Keypair.from_seed(get_key_response.decode_key()) -def to_keypair_secure(get_key_response: GetKeyResponse | GetTlsKeyResponse) -> Keypair: - """Create a Solana Keypair using SHA256 of full key material for security.""" - if isinstance(get_key_response, GetTlsKeyResponse): - warnings.warn( - "to_keypair_secure: Please don't use getTlsKey method to get key, use getKey instead.", - DeprecationWarning, - stacklevel=2, - ) - try: - # Hash the complete key material with SHA256 - key_bytes = get_key_response.as_uint8array() - hashed_key = hashlib.sha256(key_bytes).digest() - return Keypair.from_seed(hashed_key) - except Exception as e: - raise RuntimeError( - "to_keypair_secure: missing SHA256 support, please upgrade your system" - ) from e - else: # GetKeyResponse - return Keypair.from_seed(get_key_response.decode_key()) +def to_keypair_secure(get_key_response: GetKeyResponse) -> Keypair: + """Create a Solana keypair from a DstackClientV0 get_key() response.""" + _reject_tls_key(get_key_response) + return Keypair.from_seed(get_key_response.decode_key()) diff --git a/sdk/python/tests/test_ethereum.py b/sdk/python/tests/test_ethereum.py index c270ae5bc..30aaaff01 100644 --- a/sdk/python/tests/test_ethereum.py +++ b/sdk/python/tests/test_ethereum.py @@ -2,8 +2,6 @@ # # SPDX-License-Identifier: Apache-2.0 -import warnings - from eth_account.signers.local import LocalAccount import pytest @@ -58,49 +56,10 @@ def test_sync_to_account_secure(): assert isinstance(account, LocalAccount) -def test_to_account_with_tls_key(): - """Test to_account with TLS key response (should show warning).""" +@pytest.mark.parametrize("adapter", [to_account, to_account_secure]) +def test_account_adapters_reject_tls_keys(adapter): from dstack_sdk import GetTlsKeyResponse - # Use mock TLS key response instead of actual server call - mock_result = GetTlsKeyResponse( - key="""-----BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgKONKWRjMvhgxHDmr -SY7zfjPHe3Qp8vCO9HqjzjqhXNKhRANCAAT5XHKyj7JRGHl2nQ2SltGKjQ3A7MPJ -/7JDkUxMNYhTxKqYdJZ6l1C8XrjKc7SFsVJhYgdJjLzQ3xKJz6l5jKzQ ------END PRIVATE KEY-----""", - certificate_chain=["cert1", "cert2"], - ) - - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - account = to_account(mock_result) - - assert isinstance(account, LocalAccount) - assert len(w) == 1 - assert issubclass(w[0].category, DeprecationWarning) - assert "Please don't use getTlsKey method" in str(w[0].message) - - -def test_to_account_secure_with_tls_key(): - """Test to_account_secure with TLS key response (should show warning).""" - from dstack_sdk import GetTlsKeyResponse - - # Use mock TLS key response instead of actual server call - mock_result = GetTlsKeyResponse( - key="""-----BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgKONKWRjMvhgxHDmr -SY7zfjPHe3Qp8vCO9HqjzjqhXNKhRANCAAT5XHKyj7JRGHl2nQ2SltGKjQ3A7MPJ -/7JDkUxMNYhTxKqYdJZ6l1C8XrjKc7SFsVJhYgdJjLzQ3xKJz6l5jKzQ ------END PRIVATE KEY-----""", - certificate_chain=["cert1", "cert2"], - ) - - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - account = to_account_secure(mock_result) - - assert isinstance(account, LocalAccount) - assert len(w) == 1 - assert issubclass(w[0].category, DeprecationWarning) - assert "Please don't use getTlsKey method" in str(w[0].message) + response = GetTlsKeyResponse(key="not used", certificate_chain=[]) + with pytest.raises(TypeError, match="TLS keys cannot be used"): + adapter(response) # type: ignore[arg-type] diff --git a/sdk/python/tests/test_solana.py b/sdk/python/tests/test_solana.py index 27cb9bf30..b42d6a0e3 100644 --- a/sdk/python/tests/test_solana.py +++ b/sdk/python/tests/test_solana.py @@ -2,77 +2,23 @@ # # SPDX-License-Identifier: Apache-2.0 -import warnings - import pytest from solders.keypair import Keypair -from dstack_sdk import AsyncDstackClientV0 -from dstack_sdk import DstackClientV0 from dstack_sdk import GetKeyResponse +from dstack_sdk import GetTlsKeyResponse from dstack_sdk.solana import to_keypair from dstack_sdk.solana import to_keypair_secure -@pytest.mark.asyncio -async def test_async_to_keypair(): - client = AsyncDstackClientV0() - result = await client.get_key("test") - assert isinstance(result, GetKeyResponse) - keypair = to_keypair(result) - assert isinstance(keypair, Keypair) - - -def test_sync_to_keypair(): - client = DstackClientV0() - result = client.get_key("test") - assert isinstance(result, GetKeyResponse) - keypair = to_keypair(result) - assert isinstance(keypair, Keypair) - - -@pytest.mark.asyncio -async def test_async_to_keypair_secure(): - client = AsyncDstackClientV0() - result = await client.get_key("test") - assert isinstance(result, GetKeyResponse) - keypair = to_keypair_secure(result) - assert isinstance(keypair, Keypair) - - -def test_sync_to_keypair_secure(): - client = DstackClientV0() - result = client.get_key("test") - assert isinstance(result, GetKeyResponse) - keypair = to_keypair_secure(result) - assert isinstance(keypair, Keypair) - - -def test_to_keypair_with_tls_key(): - """Test to_keypair with TLS key response (should show warning).""" - client = DstackClientV0() - result = client.get_tls_key() - - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - keypair = to_keypair(result) - - assert isinstance(keypair, Keypair) - assert len(w) == 1 - assert issubclass(w[0].category, DeprecationWarning) - assert "Please don't use getTlsKey method" in str(w[0].message) - - -def test_to_keypair_secure_with_tls_key(): - """Test to_keypair_secure with TLS key response (should show warning).""" - client = DstackClientV0() - result = client.get_tls_key() +@pytest.mark.parametrize("adapter", [to_keypair, to_keypair_secure]) +def test_keypair_adapters_accept_derived_keys(adapter): + response = GetKeyResponse(key="01" * 32, signature_chain=[]) + assert isinstance(adapter(response), Keypair) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - keypair = to_keypair_secure(result) - assert isinstance(keypair, Keypair) - assert len(w) == 1 - assert issubclass(w[0].category, DeprecationWarning) - assert "Please don't use getTlsKey method" in str(w[0].message) +@pytest.mark.parametrize("adapter", [to_keypair, to_keypair_secure]) +def test_keypair_adapters_reject_tls_keys(adapter): + response = GetTlsKeyResponse(key="not used", certificate_chain=[]) + with pytest.raises(TypeError, match="TLS keys cannot be used"): + adapter(response) # type: ignore[arg-type]